Project: Concrete Strength Prediction

Import packages

In [1]:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

from sklearn import linear_model
from sklearn.linear_model import LinearRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split as tts

import re
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import scale
from sklearn.feature_selection import RFE
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import KFold
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import make_pipeline

from mpl_toolkits.mplot3d import Axes3D

from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestRegressor
from sklearn import tree
from sklearn.tree import DecisionTreeRegressor
from IPython.display import Image
from os import system

from sklearn.model_selection import cross_validate
from sklearn.model_selection import RandomizedSearchCV

from IPython.core.interactiveshell import InteractiveShell

# Typically would use inline
# but doing this for a 3d chart
# in the bivariate analysis section
%matplotlib inline

InteractiveShell.ast_node_interactivity = 'all'

plt.rc('figure', max_open_warning=0)

Set package options

In [2]:
sns.set(color_codes=True)
sns.set_style(style='darkgrid')
palette = 'Set2'
pd.set_option('display.max_columns', None)

Import dataset; one for raw data, one to edited/imputed data

In [3]:
raw_data = pd.read_csv('concrete.csv')
data = pd.read_csv('concrete.csv')

Methods for use elsewhere

In [4]:
figx = 10
figy = 8

def dist(col):
    plt.figure(figsize=(figx,figy))
    sns.distplot(col);
    
def hist(col):
    plt.figure(figsize=(figx,figy))
    plt.hist(col)
    plt.axvline(col.mean(), color='y', linewidth=2, label='Mean')
    plt.axvline(col.median(), color='g', linewidth=2, label='Median')
    plt.legend();
    
def box(col):
    plt.figure(figsize=(figx,figy))
    sns.boxplot(col);
    
def print_summary(col):
    dist(col)
    hist(col)
    box(col)
    print(col.describe())
    print('')
    print('Unique values: ' + str(col.nunique()))
    
def replace_zeros(col, val):
    data[col].replace(0, val, inplace=True)
    
def marginal_boxplot_margins(a, vertical=False, **kws):
    if vertical:
        sns.boxplot(y=a, palette='Accent', **kws)
    else:
        sns.boxplot(x=a, palette='Accent_r', **kws)

def marginal_boxplot(xcol, ycol, raw=True):
    if raw:
        g = sns.JointGrid(data=raw_data, x=xcol, y=ycol);
    else:
        g = sns.JointGrid(data=data, x=xcol, y=ycol);
    g.plot_joint(sns.regplot, lowess=True, truncate=False, scatter_kws={'alpha':.2});
    g.plot_marginals(marginal_boxplot_margins);

Generic data analysis

In [5]:
raw_data.head(10)
Out[5]:
cement slag ash water superplastic coarseagg fineagg age strength
0 141.3 212.0 0.0 203.5 0.0 971.8 748.5 28 29.89
1 168.9 42.2 124.3 158.3 10.8 1080.8 796.2 14 23.51
2 250.0 0.0 95.7 187.4 5.5 956.9 861.2 28 29.22
3 266.0 114.0 0.0 228.0 0.0 932.0 670.0 28 45.85
4 154.8 183.4 0.0 193.3 9.1 1047.4 696.7 28 18.29
5 255.0 0.0 0.0 192.0 0.0 889.8 945.0 90 21.86
6 166.8 250.2 0.0 203.5 0.0 975.6 692.6 7 15.75
7 251.4 0.0 118.3 188.5 6.4 1028.4 757.7 56 36.64
8 296.0 0.0 0.0 192.0 0.0 1085.0 765.0 28 21.65
9 155.0 184.0 143.0 194.0 9.0 880.0 699.0 28 28.99
In [6]:
raw_data.shape
Out[6]:
(1030, 9)
In [7]:
raw_data.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1030 entries, 0 to 1029
Data columns (total 9 columns):
 #   Column        Non-Null Count  Dtype  
---  ------        --------------  -----  
 0   cement        1030 non-null   float64
 1   slag          1030 non-null   float64
 2   ash           1030 non-null   float64
 3   water         1030 non-null   float64
 4   superplastic  1030 non-null   float64
 5   coarseagg     1030 non-null   float64
 6   fineagg       1030 non-null   float64
 7   age           1030 non-null   int64  
 8   strength      1030 non-null   float64
dtypes: float64(8), int64(1)
memory usage: 72.5 KB
In [8]:
raw_data.describe()
Out[8]:
cement slag ash water superplastic coarseagg fineagg age strength
count 1030.000000 1030.000000 1030.000000 1030.000000 1030.000000 1030.000000 1030.000000 1030.000000 1030.000000
mean 281.167864 73.895825 54.188350 181.567282 6.204660 972.918932 773.580485 45.662136 35.817961
std 104.506364 86.279342 63.997004 21.354219 5.973841 77.753954 80.175980 63.169912 16.705742
min 102.000000 0.000000 0.000000 121.800000 0.000000 801.000000 594.000000 1.000000 2.330000
25% 192.375000 0.000000 0.000000 164.900000 0.000000 932.000000 730.950000 7.000000 23.710000
50% 272.900000 22.000000 0.000000 185.000000 6.400000 968.000000 779.500000 28.000000 34.445000
75% 350.000000 142.950000 118.300000 192.000000 10.200000 1029.400000 824.000000 56.000000 46.135000
max 540.000000 359.400000 200.100000 247.000000 32.200000 1145.000000 992.600000 365.000000 82.600000

Univariate analysis

Cement (cement)

In [9]:
print_summary(raw_data['cement'])
count    1030.000000
mean      281.167864
std       104.506364
min       102.000000
25%       192.375000
50%       272.900000
75%       350.000000
max       540.000000
Name: cement, dtype: float64

Unique values: 278

Distribution of cement has a very slight right-hand skew.

Blast Furance Slag (slag)

In [10]:
print_summary(raw_data['slag'])
count    1030.000000
mean       73.895825
std        86.279342
min         0.000000
25%         0.000000
50%        22.000000
75%       142.950000
max       359.400000
Name: slag, dtype: float64

Unique values: 185
In [11]:
replace_zeros('slag', np.nan)

print_summary(data['slag'])
C:\Users\pcopley\Anaconda3\lib\site-packages\numpy\lib\histograms.py:839: RuntimeWarning: invalid value encountered in greater_equal
  keep = (tmp_a >= first_edge)
C:\Users\pcopley\Anaconda3\lib\site-packages\numpy\lib\histograms.py:840: RuntimeWarning: invalid value encountered in less_equal
  keep &= (tmp_a <= last_edge)
count    559.000000
mean     136.158676
std       72.351823
min       11.000000
25%       95.000000
50%      135.700000
75%      189.000000
max      359.400000
Name: slag, dtype: float64

Unique values: 184

Distribution of slag has a very large right hand skew.

Fly Ash (ash)

In [12]:
print_summary(raw_data['ash'])
count    1030.000000
mean       54.188350
std        63.997004
min         0.000000
25%         0.000000
50%         0.000000
75%       118.300000
max       200.100000
Name: ash, dtype: float64

Unique values: 156

Distribution of ash has a very strong right-hand skew, however this is largely due to most samples have no ash content.

What does it look like if we only look at samples containing some ash?

In [13]:
replace_zeros('ash', np.nan)

print_summary(data['ash'])
C:\Users\pcopley\Anaconda3\lib\site-packages\numpy\lib\histograms.py:839: RuntimeWarning: invalid value encountered in greater_equal
  keep = (tmp_a >= first_edge)
C:\Users\pcopley\Anaconda3\lib\site-packages\numpy\lib\histograms.py:840: RuntimeWarning: invalid value encountered in less_equal
  keep &= (tmp_a <= last_edge)
count    464.000000
mean     120.288793
std       33.675470
min       24.500000
25%       97.850000
50%      121.400000
75%      141.000000
max      200.100000
Name: ash, dtype: float64

Unique values: 155

If we look at the distribution of ash in samples with non-zero levels of ash, we see a moderate left-hand skew.

Water (water)

In [14]:
print_summary(raw_data['water'])
count    1030.000000
mean      181.567282
std        21.354219
min       121.800000
25%       164.900000
50%       185.000000
75%       192.000000
max       247.000000
Name: water, dtype: float64

Unique values: 195

Distribution of water has a strong left-hand skew but also several large outliers.

Superplasticizer (superplastic)

In [15]:
print_summary(raw_data['superplastic'])
count    1030.000000
mean        6.204660
std         5.973841
min         0.000000
25%         0.000000
50%         6.400000
75%        10.200000
max        32.200000
Name: superplastic, dtype: float64

Unique values: 111

Distribution of superplastic has a strong right skew with a few large outliers. It also has many 0-values so let's look at what the non-zero superplastic distribution looks like.

In [16]:
replace_zeros('superplastic', np.nan)

print_summary(data['superplastic'])
C:\Users\pcopley\Anaconda3\lib\site-packages\numpy\lib\histograms.py:839: RuntimeWarning: invalid value encountered in greater_equal
  keep = (tmp_a >= first_edge)
C:\Users\pcopley\Anaconda3\lib\site-packages\numpy\lib\histograms.py:840: RuntimeWarning: invalid value encountered in less_equal
  keep &= (tmp_a <= last_edge)
count    651.000000
mean       9.816897
std        4.580328
min        1.700000
25%        6.950000
50%        9.400000
75%       11.600000
max       32.200000
Name: superplastic, dtype: float64

Unique values: 110

Looking at only non-zero superplastic values, we see a right-hand skew with several outliers.

Coarse Aggregate (coarseagg)

In [17]:
print_summary(raw_data['coarseagg'])
count    1030.000000
mean      972.918932
std        77.753954
min       801.000000
25%       932.000000
50%       968.000000
75%      1029.400000
max      1145.000000
Name: coarseagg, dtype: float64

Unique values: 284

Distribution of courseagg has a slight right skew and no outliers.

Fine Aggregate (fineagg)

In [18]:
print_summary(raw_data['fineagg'])
count    1030.000000
mean      773.580485
std        80.175980
min       594.000000
25%       730.950000
50%       779.500000
75%       824.000000
max       992.600000
Name: fineagg, dtype: float64

Unique values: 302

Distribution of fineagg does not have a noticeable skew.

Age (age)

In [19]:
print_summary(raw_data['age'])
count    1030.000000
mean       45.662136
std        63.169912
min         1.000000
25%         7.000000
50%        28.000000
75%        56.000000
max       365.000000
Name: age, dtype: float64

Unique values: 14

Distribution of age has a right skew with several large outliers for particularly older samples.

Compressive Strength (strength)

In [20]:
print_summary(raw_data['strength'])
count    1030.000000
mean       35.817961
std        16.705742
min         2.330000
25%        23.710000
50%        34.445000
75%        46.135000
max        82.600000
Name: strength, dtype: float64

Unique values: 845

Distribution of strength has a slight right skew with a few large outliers.

Bivariate analyses

In [21]:
g = sns.PairGrid(raw_data);

g.map_upper(sns.scatterplot);
g.map_diag(sns.kdeplot);
g.map_lower(sns.kdeplot);
In [22]:
plt.figure(figsize=(15,10))
sns.heatmap(raw_data.corr(), annot=True, cmap='mako', vmin=-.658, vmax=.498, center=0, fmt='.3f');

With the raw data, the strongest positive correlations seem to be between (1) strength and cement content, (2) ash content and superplastic content, and (3) superplastic content and strength.

In [23]:
marginal_boxplot('strength', 'cement')
In [24]:
marginal_boxplot('ash', 'superplastic')
In [25]:
marginal_boxplot('strength', 'superplastic')

The strongest negative correlations seem to be between (1) superplastic content and water content, (2) fine aggregate content and water content, and (3) cement content and ash content.

In [26]:
marginal_boxplot('superplastic', 'water')
In [27]:
marginal_boxplot('fineagg', 'water')
In [28]:
marginal_boxplot('cement', 'ash')
In [29]:
plt.figure(figsize=(15,10))
sns.heatmap(data.corr(), annot=True, cmap='mako', vmin=-.538, vmax=.498, center=0, fmt='.3f');
In [30]:
# Re-run the marginal boxplots for the non-zero values just to see if there is any significant difference
marginal_boxplot('ash', 'superplastic')
marginal_boxplot('ash', 'superplastic', raw=False)
In [31]:
marginal_boxplot('strength', 'superplastic')
marginal_boxplot('strength', 'superplastic', raw=False)
In [32]:
marginal_boxplot('superplastic', 'water')
marginal_boxplot('superplastic', 'water', raw=False)
In [33]:
marginal_boxplot('cement', 'ash')
marginal_boxplot('cement', 'ash', raw=False)

There does not seem to be any significant change in correlation when we remove zero values from the raw data.

The strongest positive correlations are centered around cement content, superplastics content, and strength. Another is ash, which is positively correlated with superplastics but negatively correlated with strength. So what does the relationship between cement, superplastics, and ash look like?

For this we'll utilize a 3-dimensional scatterplot; one with the zero values, one without.

In [34]:
fig = plt.figure(figsize=(25,15))
ax = fig.add_subplot(111, projection='3d')

x = raw_data['cement']
y = raw_data['superplastic']
z = raw_data['ash']

ax.scatter(x, y, z, c='r', marker='o')

ax.set_xlabel('Cement Content')
ax.set_ylabel('Superplastics Content')
ax.set_zlabel('Ash Content')

plt.show();
In [35]:
from mpl_toolkits.mplot3d import Axes3D

fig = plt.figure(figsize=(25,15))
ax = fig.add_subplot(111, projection='3d')

x = data['cement']
y = data['superplastic']
z = data['ash']

ax.scatter(x, y, z, c='r', marker='o')

ax.set_xlabel('Cement Content')
ax.set_ylabel('Superplastics Content')
ax.set_zlabel('Ash Content')

plt.show();

Feature Engineering Techniques

Feature Extraction

In [36]:
raw_data.head()
Out[36]:
cement slag ash water superplastic coarseagg fineagg age strength
0 141.3 212.0 0.0 203.5 0.0 971.8 748.5 28 29.89
1 168.9 42.2 124.3 158.3 10.8 1080.8 796.2 14 23.51
2 250.0 0.0 95.7 187.4 5.5 956.9 861.2 28 29.22
3 266.0 114.0 0.0 228.0 0.0 932.0 670.0 28 45.85
4 154.8 183.4 0.0 193.3 9.1 1047.4 696.7 28 18.29

One simple feature we can add is total aggregate content, which is just the sum of coarseagg and fineagg.

In [37]:
raw_data['agg'] = raw_data['coarseagg'] + raw_data['fineagg']
data['agg'] = data['coarseagg'] + data['fineagg']
In [38]:
raw_data.head()
Out[38]:
cement slag ash water superplastic coarseagg fineagg age strength agg
0 141.3 212.0 0.0 203.5 0.0 971.8 748.5 28 29.89 1720.3
1 168.9 42.2 124.3 158.3 10.8 1080.8 796.2 14 23.51 1877.0
2 250.0 0.0 95.7 187.4 5.5 956.9 861.2 28 29.22 1818.1
3 266.0 114.0 0.0 228.0 0.0 932.0 670.0 28 45.85 1602.0
4 154.8 183.4 0.0 193.3 9.1 1047.4 696.7 28 18.29 1744.1

Additionally, since superplasticizers are synthetic compounds designed to reduce the required amount of water, perhaps we can get something useful out of the water:superplastic ratio?

In [39]:
raw_data['swr'] = raw_data['water'] / raw_data['superplastic']
data['swr'] = data['water'] / data['superplastic']

# This is more readable than superplastic:water, but we get inf when dividing by zero, so let's replace that with nan
raw_data['swr'].replace(np.inf, np.nan, inplace=True)
data['swr'].replace(np.inf, np.nan, inplace=True)
In [40]:
raw_data.head()
Out[40]:
cement slag ash water superplastic coarseagg fineagg age strength agg swr
0 141.3 212.0 0.0 203.5 0.0 971.8 748.5 28 29.89 1720.3 NaN
1 168.9 42.2 124.3 158.3 10.8 1080.8 796.2 14 23.51 1877.0 14.657407
2 250.0 0.0 95.7 187.4 5.5 956.9 861.2 28 29.22 1818.1 34.072727
3 266.0 114.0 0.0 228.0 0.0 932.0 670.0 28 45.85 1602.0 NaN
4 154.8 183.4 0.0 193.3 9.1 1047.4 696.7 28 18.29 1744.1 21.241758

Finally, maybe our model won't care how much of something was added, just that it was? So let's create a one-hot-esque column that is just a flag for whether or not that ingredient has been added.

In [41]:
# We don't need to do this for all columns, only those that have zero values
raw_data['contains_slag'] = raw_data['slag'] > 0
raw_data['contains_ash'] = raw_data['ash'] > 0
raw_data['contains_superplastic'] = raw_data['superplastic'] > 0

data['contains_slag'] = data['slag'] > 0
data['contains_ash'] = data['ash'] > 0
data['contains_superplastic'] = data['superplastic'] > 0

# Recast True/False as 1/0
raw_data['contains_slag'] = raw_data['contains_slag'].astype(int)
raw_data['contains_ash'] = raw_data['contains_ash'].astype(int)
raw_data['contains_superplastic'] = raw_data['contains_superplastic'].astype(int)

data['contains_slag'] = data['contains_slag'].astype(int)
data['contains_ash'] = data['contains_ash'].astype(int)
data['contains_superplastic'] = data['contains_superplastic'].astype(int)
In [42]:
raw_data.head()
Out[42]:
cement slag ash water superplastic coarseagg fineagg age strength agg swr contains_slag contains_ash contains_superplastic
0 141.3 212.0 0.0 203.5 0.0 971.8 748.5 28 29.89 1720.3 NaN 1 0 0
1 168.9 42.2 124.3 158.3 10.8 1080.8 796.2 14 23.51 1877.0 14.657407 1 1 1
2 250.0 0.0 95.7 187.4 5.5 956.9 861.2 28 29.22 1818.1 34.072727 0 1 1
3 266.0 114.0 0.0 228.0 0.0 932.0 670.0 28 45.85 1602.0 NaN 1 0 0
4 154.8 183.4 0.0 193.3 9.1 1047.4 696.7 28 18.29 1744.1 21.241758 1 0 1
In [43]:
data.head()
Out[43]:
cement slag ash water superplastic coarseagg fineagg age strength agg swr contains_slag contains_ash contains_superplastic
0 141.3 212.0 NaN 203.5 NaN 971.8 748.5 28 29.89 1720.3 NaN 1 0 0
1 168.9 42.2 124.3 158.3 10.8 1080.8 796.2 14 23.51 1877.0 14.657407 1 1 1
2 250.0 NaN 95.7 187.4 5.5 956.9 861.2 28 29.22 1818.1 34.072727 0 1 1
3 266.0 114.0 NaN 228.0 NaN 932.0 670.0 28 45.85 1602.0 NaN 1 0 0
4 154.8 183.4 NaN 193.3 9.1 1047.4 696.7 28 18.29 1744.1 21.241758 1 0 1

Train/Test Split

In [44]:
# LinearRegression doesn't like np.nan, so we'll use raw_data
# and replace swr nan's with 0

raw_data['swr'].replace(np.nan, 0, inplace=True)
x = raw_data.drop('strength', axis=1)
y = raw_data[['strength']]
In [45]:
train_ratio = .7

x_train, x_test, y_train, y_test = tts(x, y, test_size=1-train_ratio, random_state=1)
In [46]:
labels = ['Training Score', 'Testing Score', '10-Fold CV Mean', '10-Fold CV Std. Dev.']
In [47]:
simple_linear_regression = LinearRegression()
simple_linear_regression.fit(x_train, y_train)
Out[47]:
LinearRegression()
In [48]:
for idx, col_name in enumerate(x_train.columns):
    print("The coefficient for {} is {}.".format(col_name, simple_linear_regression.coef_[0][idx]))
The coefficient for cement is 0.11411851032308984.
The coefficient for slag is 0.0886246888240249.
The coefficient for ash is 0.015494339318093343.
The coefficient for water is -0.1260184282008129.
The coefficient for superplastic is -0.5300383276953631.
The coefficient for coarseagg is 818789528842.1881.
The coefficient for fineagg is 818789528842.1982.
The coefficient for age is 0.1141598653445305.
The coefficient for agg is -818789528842.1746.
The coefficient for swr is -0.11369922898004697.
The coefficient for contains_slag is -0.05722979299054391.
The coefficient for contains_ash is 1.7396467070143116.
The coefficient for contains_superplastic is 18.73162969685704.
In [49]:
intercept = simple_linear_regression.intercept_[0]
print("The intercept for our model is {}".format(intercept))
The intercept for our model is -25.35044444444444
In [50]:
slr_train = simple_linear_regression.score(x_train, y_train)
slr_test = simple_linear_regression.score(x_test, y_test)
In [51]:
slr_cv = cross_validate(simple_linear_regression, x, y.values.ravel(), cv=10)

linear_reg_series = pd.Series(data=[slr_train, slr_test, slr_cv['test_score'].mean(), slr_cv['test_score'].std()],
                             index=labels)

model_comparison = pd.DataFrame(linear_reg_series, columns=['Simple Linear Regression'])

model_comparison.T
Out[51]:
Training Score Testing Score 10-Fold CV Mean 10-Fold CV Std. Dev.
Simple Linear Regression 0.645278 0.648877 0.622869 0.076005

Clearly with a dataset this complicated, linear regression isn't going to cut it. Let's see if a decision tree performs any better.

In [52]:
dt = DecisionTreeRegressor(random_state=1)
dt.fit(x_train, y_train)
Out[52]:
DecisionTreeRegressor(random_state=1)
In [53]:
dt_train = dt.score(x_train, y_train)
dt_test = dt.score(x_test, y_test)

dt_cv = cross_validate(dt, x, y.values.ravel(), cv=10)
In [54]:
model_comparison['Decision Tree'] = pd.Series(data=[dt_train, dt_test, dt_cv['test_score'].mean(), dt_cv['test_score'].std()],
                                             index=labels)

model_comparison.T
Out[54]:
Training Score Testing Score 10-Fold CV Mean 10-Fold CV Std. Dev.
Simple Linear Regression 0.645278 0.648877 0.622869 0.076005
Decision Tree 0.994855 0.846844 0.861023 0.048994
In [55]:
pd.DataFrame(dt.feature_importances_, columns=['Imp'], index=x_train.columns)
Out[55]:
Imp
cement 0.344124
slag 0.089350
ash 0.019093
water 0.118810
superplastic 0.011971
coarseagg 0.025242
fineagg 0.006602
age 0.337260
agg 0.036446
swr 0.009485
contains_slag 0.000034
contains_ash 0.001421
contains_superplastic 0.000162

Much better performance than linear, but how much better will RF perform?

In [56]:
# max_features=8 gives us the best test score of 90.92
rf = RandomForestRegressor(n_estimators=50, random_state=1, max_features=8)
rf = rf.fit(x_train, y_train.values.ravel())
In [57]:
rf_train = rf.score(x_train, y_train)
rf_test = rf.score(x_test, y_test)

rf_cv = cross_validate(rf, x, y.values.ravel(), cv=10)
In [58]:
model_comparison['Random Forest'] = pd.Series(data=[rf_train, rf_test, rf_cv['test_score'].mean(), rf_cv['test_score'].std()],
                                             index=labels)

model_comparison.T
Out[58]:
Training Score Testing Score 10-Fold CV Mean 10-Fold CV Std. Dev.
Simple Linear Regression 0.645278 0.648877 0.622869 0.076005
Decision Tree 0.994855 0.846844 0.861023 0.048994
Random Forest 0.982552 0.909249 0.919497 0.024503

MinMaxScaler

In [59]:
df_columns = raw_data.columns.drop('strength')
scaler = MinMaxScaler()
df = scaler.fit_transform(raw_data.drop('strength', axis=1))

df = pd.DataFrame(df)
df.columns = df_columns
df['strength'] = raw_data['strength']

df.head()
Out[59]:
cement slag ash water superplastic coarseagg fineagg age agg swr contains_slag contains_ash contains_superplastic strength
0 0.089726 0.589872 0.000000 0.652556 0.000000 0.496512 0.387607 0.074176 0.513255 0.000000 1.0 0.0 0.0 29.89
1 0.152740 0.117418 0.621189 0.291534 0.335404 0.813372 0.507275 0.035714 0.818713 0.120959 1.0 1.0 1.0 23.51
2 0.337900 0.000000 0.478261 0.523962 0.170807 0.453198 0.670346 0.074176 0.703899 0.281183 0.0 1.0 1.0 29.22
3 0.374429 0.317195 0.000000 0.848243 0.000000 0.380814 0.190667 0.074176 0.282651 0.000000 1.0 0.0 0.0 45.85
4 0.120548 0.510295 0.000000 0.571086 0.282609 0.716279 0.257652 0.074176 0.559649 0.175296 1.0 0.0 1.0 18.29
In [60]:
x = df.drop('strength', axis=1)
y = df[['strength']]
x_train, x_test, y_train, y_test = tts(x, y, test_size=1-train_ratio, random_state=1)

# max_features=6 gives us the best test score of 90.67
rf2 = RandomForestRegressor(n_estimators=50, max_features=6, random_state=1)
rf2 = rf2.fit(x_train, y_train.values.ravel())

rf_mms_train = rf2.score(x_train, y_train)
rf_mms_test = rf2.score(x_test, y_test)

rf_mms_cv = cross_validate(rf2, x, y.values.ravel(), cv=10)
In [61]:
model_comparison['Random Forest with MinMaxScaler'] = pd.Series(data=[rf_mms_train, rf_mms_test, rf_mms_cv['test_score'].mean(), rf_mms_cv['test_score'].std()],
                                                                index=labels)

model_comparison.T
Out[61]:
Training Score Testing Score 10-Fold CV Mean 10-Fold CV Std. Dev.
Simple Linear Regression 0.645278 0.648877 0.622869 0.076005
Decision Tree 0.994855 0.846844 0.861023 0.048994
Random Forest 0.982552 0.909249 0.919497 0.024503
Random Forest with MinMaxScaler 0.982306 0.906661 0.920405 0.023778

Utilizing MinMaxScaler() gives us nearly identical performance, but only using 6 features instead of 8.

GradientBoostingRegressor

In [62]:
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.metrics import mean_squared_error

gbr = GradientBoostingRegressor()
gbr = gbr.fit(x_train, y_train.values.ravel())
In [63]:
gbr_train = gbr.score(x_train, y_train)
gbr_test = gbr.score(x_test, y_test)

gbr_cv = cross_validate(gbr, x, y.values.ravel(), cv=10)
In [64]:
model_comparison['Gradient Boosting Regressor'] = pd.Series(data=[gbr_train, gbr_test, gbr_cv['test_score'].mean(), gbr_cv['test_score'].std()],
                                                            index=labels)

model_comparison.T
Out[64]:
Training Score Testing Score 10-Fold CV Mean 10-Fold CV Std. Dev.
Simple Linear Regression 0.645278 0.648877 0.622869 0.076005
Decision Tree 0.994855 0.846844 0.861023 0.048994
Random Forest 0.982552 0.909249 0.919497 0.024503
Random Forest with MinMaxScaler 0.982306 0.906661 0.920405 0.023778
Gradient Boosting Regressor 0.952045 0.899542 0.904101 0.022228

RandomizedSearchCV

In [65]:
print('RandomForest parameters (no MinMaxScaler):\n')
print(rf.get_params())
RandomForest parameters (no MinMaxScaler):

{'bootstrap': True, 'ccp_alpha': 0.0, 'criterion': 'mse', 'max_depth': None, 'max_features': 8, 'max_leaf_nodes': None, 'max_samples': None, 'min_impurity_decrease': 0.0, 'min_impurity_split': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'n_estimators': 50, 'n_jobs': None, 'oob_score': False, 'random_state': 1, 'verbose': 0, 'warm_start': False}
In [66]:
random_grid = {
    'n_estimators': [int(x) for x in np.linspace(start=50, stop=2000, num=25)],
    'max_features': ['auto', 'sqrt'],
    'max_depth': [int(x) for x in np.linspace(10, 110, num=11)],
    'min_samples_split': [2, 5, 10],
    'min_samples_leaf': [1, 2, 4],
    'bootstrap': [True, False]
}

print(random_grid)
{'n_estimators': [50, 131, 212, 293, 375, 456, 537, 618, 700, 781, 862, 943, 1025, 1106, 1187, 1268, 1350, 1431, 1512, 1593, 1675, 1756, 1837, 1918, 2000], 'max_features': ['auto', 'sqrt'], 'max_depth': [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110], 'min_samples_split': [2, 5, 10], 'min_samples_leaf': [1, 2, 4], 'bootstrap': [True, False]}
In [67]:
rf_random = RandomizedSearchCV(estimator=rf, param_distributions=random_grid, n_iter=100, cv=3, verbose=2, random_state=1, n_jobs=-1)
In [68]:
rf_random.fit(x_train, y_train.values.ravel())
Fitting 3 folds for each of 100 candidates, totalling 300 fits
[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.
[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:   45.2s
[Parallel(n_jobs=-1)]: Done 154 tasks      | elapsed:  3.1min
[Parallel(n_jobs=-1)]: Done 300 out of 300 | elapsed:  6.1min finished
Out[68]:
RandomizedSearchCV(cv=3,
                   estimator=RandomForestRegressor(max_features=8,
                                                   n_estimators=50,
                                                   random_state=1),
                   n_iter=100, n_jobs=-1,
                   param_distributions={'bootstrap': [True, False],
                                        'max_depth': [10, 20, 30, 40, 50, 60,
                                                      70, 80, 90, 100, 110],
                                        'max_features': ['auto', 'sqrt'],
                                        'min_samples_leaf': [1, 2, 4],
                                        'min_samples_split': [2, 5, 10],
                                        'n_estimators': [50, 131, 212, 293, 375,
                                                         456, 537, 618, 700,
                                                         781, 862, 943, 1025,
                                                         1106, 1187, 1268, 1350,
                                                         1431, 1512, 1593, 1675,
                                                         1756, 1837, 1918,
                                                         2000]},
                   random_state=1, verbose=2)
In [69]:
rf_random.best_params_
Out[69]:
{'n_estimators': 1756,
 'min_samples_split': 2,
 'min_samples_leaf': 1,
 'max_features': 'auto',
 'max_depth': 110,
 'bootstrap': True}
In [70]:
rf_best_random = rf_random.best_estimator_
In [71]:
def evaluate(model, test_features, test_labels):
    predictions = model.predict(test_features)
    errors = abs(predictions - test_labels)
    mape = 100 * np.mean(errors / test_labels)
    accuracy = 100 - mape
    print('Model Performance')
    print('Average Error: {:0.4f} degrees.'.format(np.mean(errors)))
    print('Accuracy: {:0.2f}%.'.format(accuracy))
    
    return accuracy
In [72]:
base_model = RandomForestRegressor(n_estimators=50, random_state=1, max_features=8)
base_model.fit(x_train, y_train.values.ravel())

base_model.score(x_test, y_test.values.ravel())
Out[72]:
RandomForestRegressor(max_features=8, n_estimators=50, random_state=1)
Out[72]:
0.9091278108955855
In [73]:
rf_rscv_train = rf_best_random.score(x_train, y_train.values.ravel())
rf_rscv_test = rf_best_random.score(x_test, y_test.values.ravel())

rfrs_cv = cross_validate(rf_best_random, x, y.values.ravel(), cv=10)
In [74]:
model_comparison['Randomized Search CV'] = pd.Series(data=[rf_rscv_train, rf_rscv_test, rfrs_cv['test_score'].mean(), rfrs_cv['test_score'].std()],
                                                     index=labels)

model_comparison.T
Out[74]:
Training Score Testing Score 10-Fold CV Mean 10-Fold CV Std. Dev.
Simple Linear Regression 0.645278 0.648877 0.622869 0.076005
Decision Tree 0.994855 0.846844 0.861023 0.048994
Random Forest 0.982552 0.909249 0.919497 0.024503
Random Forest with MinMaxScaler 0.982306 0.906661 0.920405 0.023778
Gradient Boosting Regressor 0.952045 0.899542 0.904101 0.022228
Randomized Search CV 0.983311 0.905970 0.917427 0.024922

GridSearchCV

In [75]:
param_grid = {
    'bootstrap': [True],
    'max_depth': [100, 110, 120],
    'max_features': ['auto'],
    'min_samples_leaf': [1, 2],
    'min_samples_split': [1, 2, 3],
    'n_estimators': [1650, 1750, 1850]
}

grid_model_base = RandomForestRegressor(random_state=1)

grid_search = GridSearchCV(estimator=grid_model_base, param_grid=param_grid, cv=3, n_jobs=-1, verbose=2)
In [76]:
grid_search.fit(x_train, y_train.values.ravel())
Fitting 3 folds for each of 54 candidates, totalling 162 fits
[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.
[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:   49.4s
[Parallel(n_jobs=-1)]: Done 154 tasks      | elapsed:  4.3min
[Parallel(n_jobs=-1)]: Done 162 out of 162 | elapsed:  4.6min finished
Out[76]:
GridSearchCV(cv=3, estimator=RandomForestRegressor(random_state=1), n_jobs=-1,
             param_grid={'bootstrap': [True], 'max_depth': [100, 110, 120],
                         'max_features': ['auto'], 'min_samples_leaf': [1, 2],
                         'min_samples_split': [1, 2, 3],
                         'n_estimators': [1650, 1750, 1850]},
             verbose=2)
In [77]:
grid_search.best_params_
Out[77]:
{'bootstrap': True,
 'max_depth': 100,
 'max_features': 'auto',
 'min_samples_leaf': 1,
 'min_samples_split': 2,
 'n_estimators': 1850}
In [78]:
rf_gscv_train = grid_search.best_estimator_.score(x_train, y_train.values.ravel())
rf_gscv_test = grid_search.best_estimator_.score(x_test, y_test.values.ravel())

rfgs_cv = cross_validate(grid_search.best_estimator_, x, y.values.ravel(), cv=10)
In [79]:
model_comparison['Grid Search CV'] = pd.Series(data=[rf_gscv_train, rf_gscv_test, rfgs_cv['test_score'].mean(), rfgs_cv['test_score'].std()],
                                               index=labels)

model_comparison.T
Out[79]:
Training Score Testing Score 10-Fold CV Mean 10-Fold CV Std. Dev.
Simple Linear Regression 0.645278 0.648877 0.622869 0.076005
Decision Tree 0.994855 0.846844 0.861023 0.048994
Random Forest 0.982552 0.909249 0.919497 0.024503
Random Forest with MinMaxScaler 0.982306 0.906661 0.920405 0.023778
Gradient Boosting Regressor 0.952045 0.899542 0.904101 0.022228
Randomized Search CV 0.983311 0.905970 0.917427 0.024922
Grid Search CV 0.983307 0.905936 0.917525 0.024961

XGBoost

In [80]:
from xgboost import XGBRegressor
In [81]:
xgb = XGBRegressor()

xgb.fit(x_train, y_train.values.ravel())
Out[81]:
XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
             colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,
             importance_type='gain', interaction_constraints='',
             learning_rate=0.300000012, max_delta_step=0, max_depth=6,
             min_child_weight=1, missing=nan, monotone_constraints='()',
             n_estimators=100, n_jobs=0, num_parallel_tree=1, random_state=0,
             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,
             tree_method='exact', validate_parameters=1, verbosity=None)
In [82]:
xgb_train = xgb.score(x_train, y_train.values.ravel())
xgb_test = xgb.score(x_test, y_test.values.ravel())

xgb_cv = cross_validate(xgb, x, y.values.ravel(), cv=10)
In [83]:
model_comparison['XGBoost'] = pd.Series(data=[xgb_train, xgb_test, xgb_cv['test_score'].mean(), xgb_cv['test_score'].std()],
                                       index=labels)

model_comparison.T
Out[83]:
Training Score Testing Score 10-Fold CV Mean 10-Fold CV Std. Dev.
Simple Linear Regression 0.645278 0.648877 0.622869 0.076005
Decision Tree 0.994855 0.846844 0.861023 0.048994
Random Forest 0.982552 0.909249 0.919497 0.024503
Random Forest with MinMaxScaler 0.982306 0.906661 0.920405 0.023778
Gradient Boosting Regressor 0.952045 0.899542 0.904101 0.022228
Randomized Search CV 0.983311 0.905970 0.917427 0.024922
Grid Search CV 0.983307 0.905936 0.917525 0.024961
XGBoost 0.994335 0.916113 0.933395 0.027554
In [84]:
from xgboost import plot_importance

plot_importance(xgb)
plt.show()
Out[84]:
<matplotlib.axes._subplots.AxesSubplot at 0x19f745e8280>
In [85]:
from xgboost import plot_tree

plot_tree(xgb, num_trees=1)
fig = plt.gcf()
fig.set_size_inches(75,75);
In [86]:
xgb.get_params()
Out[86]:
{'objective': 'reg:squarederror',
 'base_score': 0.5,
 'booster': 'gbtree',
 'colsample_bylevel': 1,
 'colsample_bynode': 1,
 'colsample_bytree': 1,
 'gamma': 0,
 'gpu_id': -1,
 'importance_type': 'gain',
 'interaction_constraints': '',
 'learning_rate': 0.300000012,
 'max_delta_step': 0,
 'max_depth': 6,
 'min_child_weight': 1,
 'missing': nan,
 'monotone_constraints': '()',
 'n_estimators': 100,
 'n_jobs': 0,
 'num_parallel_tree': 1,
 'random_state': 0,
 'reg_alpha': 0,
 'reg_lambda': 1,
 'scale_pos_weight': 1,
 'subsample': 1,
 'tree_method': 'exact',
 'validate_parameters': 1,
 'verbosity': None}
In [87]:
from scipy import stats

otl = stats.beta(10,1)
zp = stats.expon(0,50)

params = {
    "n_estimators": stats.randint(3,40),
    "max_depth": stats.randint(3,40),
    "learning_rate": stats.uniform(.05, .4),
    "colsample_bytree": otl,
    "subsample": otl,
    "gamma": stats.uniform(0,10),
    "reg_alpha": zp,
    "min_child_weight": zp
}

xgbreg = XGBRegressor()
In [88]:
xgbrs = RandomizedSearchCV(n_iter=100, cv=3, verbose=2, random_state=1, estimator=xgbreg, param_distributions=params, n_jobs=1)
xgbrs.fit(x_train, y_train)
Fitting 3 folds for each of 100 candidates, totalling 300 fits
[CV] colsample_bytree=0.9775097845509342, gamma=1.4675589081711304, learning_rate=0.08693543790751912, max_depth=15, min_child_weight=24.543857966503296, n_estimators=9, reg_alpha=137.08479938255715, subsample=0.9545149598867679 
[CV]  colsample_bytree=0.9775097845509342, gamma=1.4675589081711304, learning_rate=0.08693543790751912, max_depth=15, min_child_weight=24.543857966503296, n_estimators=9, reg_alpha=137.08479938255715, subsample=0.9545149598867679, total=   0.0s
[CV] colsample_bytree=0.9775097845509342, gamma=1.4675589081711304, learning_rate=0.08693543790751912, max_depth=15, min_child_weight=24.543857966503296, n_estimators=9, reg_alpha=137.08479938255715, subsample=0.9545149598867679 
[CV]  colsample_bytree=0.9775097845509342, gamma=1.4675589081711304, learning_rate=0.08693543790751912, max_depth=15, min_child_weight=24.543857966503296, n_estimators=9, reg_alpha=137.08479938255715, subsample=0.9545149598867679, total=   0.0s
[CV] colsample_bytree=0.9775097845509342, gamma=1.4675589081711304, learning_rate=0.08693543790751912, max_depth=15, min_child_weight=24.543857966503296, n_estimators=9, reg_alpha=137.08479938255715, subsample=0.9545149598867679 
[CV]  colsample_bytree=0.9775097845509342, gamma=1.4675589081711304, learning_rate=0.08693543790751912, max_depth=15, min_child_weight=24.543857966503296, n_estimators=9, reg_alpha=137.08479938255715, subsample=0.9545149598867679, total=   0.0s
[CV] colsample_bytree=0.8223966978097618, gamma=9.13962024579233, learning_rate=0.23288192319479534, max_depth=33, min_child_weight=11.038661205836513, n_estimators=16, reg_alpha=62.93386145983712, subsample=0.9929462979076537 
[CV]  colsample_bytree=0.8223966978097618, gamma=9.13962024579233, learning_rate=0.23288192319479534, max_depth=33, min_child_weight=11.038661205836513, n_estimators=16, reg_alpha=62.93386145983712, subsample=0.9929462979076537, total=   0.0s
[CV] colsample_bytree=0.8223966978097618, gamma=9.13962024579233, learning_rate=0.23288192319479534, max_depth=33, min_child_weight=11.038661205836513, n_estimators=16, reg_alpha=62.93386145983712, subsample=0.9929462979076537 
[CV]  colsample_bytree=0.8223966978097618, gamma=9.13962024579233, learning_rate=0.23288192319479534, max_depth=33, min_child_weight=11.038661205836513, n_estimators=16, reg_alpha=62.93386145983712, subsample=0.9929462979076537, total=   0.0s
[CV] colsample_bytree=0.8223966978097618, gamma=9.13962024579233, learning_rate=0.23288192319479534, max_depth=33, min_child_weight=11.038661205836513, n_estimators=16, reg_alpha=62.93386145983712, subsample=0.9929462979076537 
[CV]  colsample_bytree=0.8223966978097618, gamma=9.13962024579233, learning_rate=0.23288192319479534, max_depth=33, min_child_weight=11.038661205836513, n_estimators=16, reg_alpha=62.93386145983712, subsample=0.9929462979076537, total=   0.0s
[CV] colsample_bytree=0.8846904207054818, gamma=2.7304997421674737, learning_rate=0.0736972805206254, max_depth=11, min_child_weight=5.1762674654770375, n_estimators=6, reg_alpha=55.68438226239254, subsample=0.9781240906789986 
[CV]  colsample_bytree=0.8846904207054818, gamma=2.7304997421674737, learning_rate=0.0736972805206254, max_depth=11, min_child_weight=5.1762674654770375, n_estimators=6, reg_alpha=55.68438226239254, subsample=0.9781240906789986, total=   0.0s
[CV] colsample_bytree=0.8846904207054818, gamma=2.7304997421674737, learning_rate=0.0736972805206254, max_depth=11, min_child_weight=5.1762674654770375, n_estimators=6, reg_alpha=55.68438226239254, subsample=0.9781240906789986 
[CV]  colsample_bytree=0.8846904207054818, gamma=2.7304997421674737, learning_rate=0.0736972805206254, max_depth=11, min_child_weight=5.1762674654770375, n_estimators=6, reg_alpha=55.68438226239254, subsample=0.9781240906789986, total=   0.0s
[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.
[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s
[CV] colsample_bytree=0.8846904207054818, gamma=2.7304997421674737, learning_rate=0.0736972805206254, max_depth=11, min_child_weight=5.1762674654770375, n_estimators=6, reg_alpha=55.68438226239254, subsample=0.9781240906789986 
[CV]  colsample_bytree=0.8846904207054818, gamma=2.7304997421674737, learning_rate=0.0736972805206254, max_depth=11, min_child_weight=5.1762674654770375, n_estimators=6, reg_alpha=55.68438226239254, subsample=0.9781240906789986, total=   0.0s
[CV] colsample_bytree=0.934051018182523, gamma=0.3417131118583905, learning_rate=0.29961199459094134, max_depth=3, min_child_weight=68.94918831379002, n_estimators=37, reg_alpha=77.86109270343921, subsample=0.9337356367900251 
[CV]  colsample_bytree=0.934051018182523, gamma=0.3417131118583905, learning_rate=0.29961199459094134, max_depth=3, min_child_weight=68.94918831379002, n_estimators=37, reg_alpha=77.86109270343921, subsample=0.9337356367900251, total=   0.0s
[CV] colsample_bytree=0.934051018182523, gamma=0.3417131118583905, learning_rate=0.29961199459094134, max_depth=3, min_child_weight=68.94918831379002, n_estimators=37, reg_alpha=77.86109270343921, subsample=0.9337356367900251 
[CV]  colsample_bytree=0.934051018182523, gamma=0.3417131118583905, learning_rate=0.29961199459094134, max_depth=3, min_child_weight=68.94918831379002, n_estimators=37, reg_alpha=77.86109270343921, subsample=0.9337356367900251, total=   0.0s
[CV] colsample_bytree=0.934051018182523, gamma=0.3417131118583905, learning_rate=0.29961199459094134, max_depth=3, min_child_weight=68.94918831379002, n_estimators=37, reg_alpha=77.86109270343921, subsample=0.9337356367900251 
[CV]  colsample_bytree=0.934051018182523, gamma=0.3417131118583905, learning_rate=0.29961199459094134, max_depth=3, min_child_weight=68.94918831379002, n_estimators=37, reg_alpha=77.86109270343921, subsample=0.9337356367900251, total=   0.0s
[CV] colsample_bytree=0.9845172071896382, gamma=0.19366957870297075, learning_rate=0.3215342131759564, max_depth=11, min_child_weight=120.64700401049453, n_estimators=13, reg_alpha=33.8216973284279, subsample=0.9304027434041872 
[CV]  colsample_bytree=0.9845172071896382, gamma=0.19366957870297075, learning_rate=0.3215342131759564, max_depth=11, min_child_weight=120.64700401049453, n_estimators=13, reg_alpha=33.8216973284279, subsample=0.9304027434041872, total=   0.0s
[CV] colsample_bytree=0.9845172071896382, gamma=0.19366957870297075, learning_rate=0.3215342131759564, max_depth=11, min_child_weight=120.64700401049453, n_estimators=13, reg_alpha=33.8216973284279, subsample=0.9304027434041872 
[CV]  colsample_bytree=0.9845172071896382, gamma=0.19366957870297075, learning_rate=0.3215342131759564, max_depth=11, min_child_weight=120.64700401049453, n_estimators=13, reg_alpha=33.8216973284279, subsample=0.9304027434041872, total=   0.0s
[CV] colsample_bytree=0.9845172071896382, gamma=0.19366957870297075, learning_rate=0.3215342131759564, max_depth=11, min_child_weight=120.64700401049453, n_estimators=13, reg_alpha=33.8216973284279, subsample=0.9304027434041872 
[CV]  colsample_bytree=0.9845172071896382, gamma=0.19366957870297075, learning_rate=0.3215342131759564, max_depth=11, min_child_weight=120.64700401049453, n_estimators=13, reg_alpha=33.8216973284279, subsample=0.9304027434041872, total=   0.0s
[CV] colsample_bytree=0.989870219416199, gamma=4.140559878195683, learning_rate=0.32776006309109806, max_depth=6, min_child_weight=38.89175481553912, n_estimators=16, reg_alpha=2.3203011748083013, subsample=0.808945392894047 
[CV]  colsample_bytree=0.989870219416199, gamma=4.140559878195683, learning_rate=0.32776006309109806, max_depth=6, min_child_weight=38.89175481553912, n_estimators=16, reg_alpha=2.3203011748083013, subsample=0.808945392894047, total=   0.0s
[CV] colsample_bytree=0.989870219416199, gamma=4.140559878195683, learning_rate=0.32776006309109806, max_depth=6, min_child_weight=38.89175481553912, n_estimators=16, reg_alpha=2.3203011748083013, subsample=0.808945392894047 
[CV]  colsample_bytree=0.989870219416199, gamma=4.140559878195683, learning_rate=0.32776006309109806, max_depth=6, min_child_weight=38.89175481553912, n_estimators=16, reg_alpha=2.3203011748083013, subsample=0.808945392894047, total=   0.0s
[CV] colsample_bytree=0.989870219416199, gamma=4.140559878195683, learning_rate=0.32776006309109806, max_depth=6, min_child_weight=38.89175481553912, n_estimators=16, reg_alpha=2.3203011748083013, subsample=0.808945392894047 
[CV]  colsample_bytree=0.989870219416199, gamma=4.140559878195683, learning_rate=0.32776006309109806, max_depth=6, min_child_weight=38.89175481553912, n_estimators=16, reg_alpha=2.3203011748083013, subsample=0.808945392894047, total=   0.0s
[CV] colsample_bytree=0.8994859359482075, gamma=5.384246942799851, learning_rate=0.31091955282541683, max_depth=21, min_child_weight=50.78297338250841, n_estimators=13, reg_alpha=107.41004682371951, subsample=0.9210458822708462 
[CV]  colsample_bytree=0.8994859359482075, gamma=5.384246942799851, learning_rate=0.31091955282541683, max_depth=21, min_child_weight=50.78297338250841, n_estimators=13, reg_alpha=107.41004682371951, subsample=0.9210458822708462, total=   0.0s
[CV] colsample_bytree=0.8994859359482075, gamma=5.384246942799851, learning_rate=0.31091955282541683, max_depth=21, min_child_weight=50.78297338250841, n_estimators=13, reg_alpha=107.41004682371951, subsample=0.9210458822708462 
[CV]  colsample_bytree=0.8994859359482075, gamma=5.384246942799851, learning_rate=0.31091955282541683, max_depth=21, min_child_weight=50.78297338250841, n_estimators=13, reg_alpha=107.41004682371951, subsample=0.9210458822708462, total=   0.0s
[CV] colsample_bytree=0.8994859359482075, gamma=5.384246942799851, learning_rate=0.31091955282541683, max_depth=21, min_child_weight=50.78297338250841, n_estimators=13, reg_alpha=107.41004682371951, subsample=0.9210458822708462 
[CV]  colsample_bytree=0.8994859359482075, gamma=5.384246942799851, learning_rate=0.31091955282541683, max_depth=21, min_child_weight=50.78297338250841, n_estimators=13, reg_alpha=107.41004682371951, subsample=0.9210458822708462, total=   0.0s
[CV] colsample_bytree=0.9170979308856062, gamma=9.648400471483855, learning_rate=0.31537659912737925, max_depth=36, min_child_weight=3.8328229919214216, n_estimators=31, reg_alpha=149.2784633224072, subsample=0.8937097060052966 
[CV]  colsample_bytree=0.9170979308856062, gamma=9.648400471483855, learning_rate=0.31537659912737925, max_depth=36, min_child_weight=3.8328229919214216, n_estimators=31, reg_alpha=149.2784633224072, subsample=0.8937097060052966, total=   0.0s
[CV] colsample_bytree=0.9170979308856062, gamma=9.648400471483855, learning_rate=0.31537659912737925, max_depth=36, min_child_weight=3.8328229919214216, n_estimators=31, reg_alpha=149.2784633224072, subsample=0.8937097060052966 
[CV]  colsample_bytree=0.9170979308856062, gamma=9.648400471483855, learning_rate=0.31537659912737925, max_depth=36, min_child_weight=3.8328229919214216, n_estimators=31, reg_alpha=149.2784633224072, subsample=0.8937097060052966, total=   0.0s
[CV] colsample_bytree=0.9170979308856062, gamma=9.648400471483855, learning_rate=0.31537659912737925, max_depth=36, min_child_weight=3.8328229919214216, n_estimators=31, reg_alpha=149.2784633224072, subsample=0.8937097060052966 
[CV]  colsample_bytree=0.9170979308856062, gamma=9.648400471483855, learning_rate=0.31537659912737925, max_depth=36, min_child_weight=3.8328229919214216, n_estimators=31, reg_alpha=149.2784633224072, subsample=0.8937097060052966, total=   0.0s
[CV] colsample_bytree=0.8725343518115356, gamma=0.0287032703115897, learning_rate=0.2968579654482896, max_depth=23, min_child_weight=43.21266499614963, n_estimators=25, reg_alpha=108.55245293714539, subsample=0.7737890364779066 
[CV]  colsample_bytree=0.8725343518115356, gamma=0.0287032703115897, learning_rate=0.2968579654482896, max_depth=23, min_child_weight=43.21266499614963, n_estimators=25, reg_alpha=108.55245293714539, subsample=0.7737890364779066, total=   0.0s
[CV] colsample_bytree=0.8725343518115356, gamma=0.0287032703115897, learning_rate=0.2968579654482896, max_depth=23, min_child_weight=43.21266499614963, n_estimators=25, reg_alpha=108.55245293714539, subsample=0.7737890364779066 
[CV]  colsample_bytree=0.8725343518115356, gamma=0.0287032703115897, learning_rate=0.2968579654482896, max_depth=23, min_child_weight=43.21266499614963, n_estimators=25, reg_alpha=108.55245293714539, subsample=0.7737890364779066, total=   0.0s
[CV] colsample_bytree=0.8725343518115356, gamma=0.0287032703115897, learning_rate=0.2968579654482896, max_depth=23, min_child_weight=43.21266499614963, n_estimators=25, reg_alpha=108.55245293714539, subsample=0.7737890364779066 
[CV]  colsample_bytree=0.8725343518115356, gamma=0.0287032703115897, learning_rate=0.2968579654482896, max_depth=23, min_child_weight=43.21266499614963, n_estimators=25, reg_alpha=108.55245293714539, subsample=0.7737890364779066, total=   0.0s
[CV] colsample_bytree=0.8900819622421665, gamma=9.973228504514806, learning_rate=0.11893620333813143, max_depth=28, min_child_weight=67.88822772352026, n_estimators=16, reg_alpha=59.67112631904906, subsample=0.873299979645472 
[CV]  colsample_bytree=0.8900819622421665, gamma=9.973228504514806, learning_rate=0.11893620333813143, max_depth=28, min_child_weight=67.88822772352026, n_estimators=16, reg_alpha=59.67112631904906, subsample=0.873299979645472, total=   0.0s
[CV] colsample_bytree=0.8900819622421665, gamma=9.973228504514806, learning_rate=0.11893620333813143, max_depth=28, min_child_weight=67.88822772352026, n_estimators=16, reg_alpha=59.67112631904906, subsample=0.873299979645472 
[CV]  colsample_bytree=0.8900819622421665, gamma=9.973228504514806, learning_rate=0.11893620333813143, max_depth=28, min_child_weight=67.88822772352026, n_estimators=16, reg_alpha=59.67112631904906, subsample=0.873299979645472, total=   0.0s
[CV] colsample_bytree=0.8900819622421665, gamma=9.973228504514806, learning_rate=0.11893620333813143, max_depth=28, min_child_weight=67.88822772352026, n_estimators=16, reg_alpha=59.67112631904906, subsample=0.873299979645472 
[CV]  colsample_bytree=0.8900819622421665, gamma=9.973228504514806, learning_rate=0.11893620333813143, max_depth=28, min_child_weight=67.88822772352026, n_estimators=16, reg_alpha=59.67112631904906, subsample=0.873299979645472, total=   0.0s
[CV] colsample_bytree=0.9872761317037585, gamma=0.19880133839795588, learning_rate=0.06048439475108772, max_depth=24, min_child_weight=35.648138769731304, n_estimators=30, reg_alpha=98.3156254875721, subsample=0.92579916634436 
[CV]  colsample_bytree=0.9872761317037585, gamma=0.19880133839795588, learning_rate=0.06048439475108772, max_depth=24, min_child_weight=35.648138769731304, n_estimators=30, reg_alpha=98.3156254875721, subsample=0.92579916634436, total=   0.0s
[CV] colsample_bytree=0.9872761317037585, gamma=0.19880133839795588, learning_rate=0.06048439475108772, max_depth=24, min_child_weight=35.648138769731304, n_estimators=30, reg_alpha=98.3156254875721, subsample=0.92579916634436 
[CV]  colsample_bytree=0.9872761317037585, gamma=0.19880133839795588, learning_rate=0.06048439475108772, max_depth=24, min_child_weight=35.648138769731304, n_estimators=30, reg_alpha=98.3156254875721, subsample=0.92579916634436, total=   0.0s
[CV] colsample_bytree=0.9872761317037585, gamma=0.19880133839795588, learning_rate=0.06048439475108772, max_depth=24, min_child_weight=35.648138769731304, n_estimators=30, reg_alpha=98.3156254875721, subsample=0.92579916634436 
[CV]  colsample_bytree=0.9872761317037585, gamma=0.19880133839795588, learning_rate=0.06048439475108772, max_depth=24, min_child_weight=35.648138769731304, n_estimators=30, reg_alpha=98.3156254875721, subsample=0.92579916634436, total=   0.0s
[CV] colsample_bytree=0.9344965699335187, gamma=0.1864728937294302, learning_rate=0.37025306907224653, max_depth=32, min_child_weight=82.28051468466589, n_estimators=23, reg_alpha=27.835610118077707, subsample=0.9074252279611857 
[CV]  colsample_bytree=0.9344965699335187, gamma=0.1864728937294302, learning_rate=0.37025306907224653, max_depth=32, min_child_weight=82.28051468466589, n_estimators=23, reg_alpha=27.835610118077707, subsample=0.9074252279611857, total=   0.0s
[CV] colsample_bytree=0.9344965699335187, gamma=0.1864728937294302, learning_rate=0.37025306907224653, max_depth=32, min_child_weight=82.28051468466589, n_estimators=23, reg_alpha=27.835610118077707, subsample=0.9074252279611857 
[CV]  colsample_bytree=0.9344965699335187, gamma=0.1864728937294302, learning_rate=0.37025306907224653, max_depth=32, min_child_weight=82.28051468466589, n_estimators=23, reg_alpha=27.835610118077707, subsample=0.9074252279611857, total=   0.0s
[CV] colsample_bytree=0.9344965699335187, gamma=0.1864728937294302, learning_rate=0.37025306907224653, max_depth=32, min_child_weight=82.28051468466589, n_estimators=23, reg_alpha=27.835610118077707, subsample=0.9074252279611857 
[CV]  colsample_bytree=0.9344965699335187, gamma=0.1864728937294302, learning_rate=0.37025306907224653, max_depth=32, min_child_weight=82.28051468466589, n_estimators=23, reg_alpha=27.835610118077707, subsample=0.9074252279611857, total=   0.0s
[CV] colsample_bytree=0.636426477222251, gamma=9.164098732731798, learning_rate=0.3627833849957864, max_depth=16, min_child_weight=62.41173340402217, n_estimators=3, reg_alpha=0.6317736291227005, subsample=0.6683557352982887 
[CV]  colsample_bytree=0.636426477222251, gamma=9.164098732731798, learning_rate=0.3627833849957864, max_depth=16, min_child_weight=62.41173340402217, n_estimators=3, reg_alpha=0.6317736291227005, subsample=0.6683557352982887, total=   0.0s
[CV] colsample_bytree=0.636426477222251, gamma=9.164098732731798, learning_rate=0.3627833849957864, max_depth=16, min_child_weight=62.41173340402217, n_estimators=3, reg_alpha=0.6317736291227005, subsample=0.6683557352982887 
[CV]  colsample_bytree=0.636426477222251, gamma=9.164098732731798, learning_rate=0.3627833849957864, max_depth=16, min_child_weight=62.41173340402217, n_estimators=3, reg_alpha=0.6317736291227005, subsample=0.6683557352982887, total=   0.0s
[CV] colsample_bytree=0.636426477222251, gamma=9.164098732731798, learning_rate=0.3627833849957864, max_depth=16, min_child_weight=62.41173340402217, n_estimators=3, reg_alpha=0.6317736291227005, subsample=0.6683557352982887 
[CV]  colsample_bytree=0.636426477222251, gamma=9.164098732731798, learning_rate=0.3627833849957864, max_depth=16, min_child_weight=62.41173340402217, n_estimators=3, reg_alpha=0.6317736291227005, subsample=0.6683557352982887, total=   0.0s
[CV] colsample_bytree=0.814884396584042, gamma=1.954294811093188, learning_rate=0.2825435709093031, max_depth=38, min_child_weight=2.565616617293441, n_estimators=37, reg_alpha=11.430305187578542, subsample=0.8269376740488578 
[CV]  colsample_bytree=0.814884396584042, gamma=1.954294811093188, learning_rate=0.2825435709093031, max_depth=38, min_child_weight=2.565616617293441, n_estimators=37, reg_alpha=11.430305187578542, subsample=0.8269376740488578, total=   0.1s
[CV] colsample_bytree=0.814884396584042, gamma=1.954294811093188, learning_rate=0.2825435709093031, max_depth=38, min_child_weight=2.565616617293441, n_estimators=37, reg_alpha=11.430305187578542, subsample=0.8269376740488578 
[CV]  colsample_bytree=0.814884396584042, gamma=1.954294811093188, learning_rate=0.2825435709093031, max_depth=38, min_child_weight=2.565616617293441, n_estimators=37, reg_alpha=11.430305187578542, subsample=0.8269376740488578, total=   0.1s
[CV] colsample_bytree=0.814884396584042, gamma=1.954294811093188, learning_rate=0.2825435709093031, max_depth=38, min_child_weight=2.565616617293441, n_estimators=37, reg_alpha=11.430305187578542, subsample=0.8269376740488578 
[CV]  colsample_bytree=0.814884396584042, gamma=1.954294811093188, learning_rate=0.2825435709093031, max_depth=38, min_child_weight=2.565616617293441, n_estimators=37, reg_alpha=11.430305187578542, subsample=0.8269376740488578, total=   0.1s
[CV] colsample_bytree=0.8778200654676123, gamma=7.625731346670644, learning_rate=0.23211298679519415, max_depth=13, min_child_weight=5.7894051446014565, n_estimators=37, reg_alpha=23.91317623249975, subsample=0.9111970919398438 
[CV]  colsample_bytree=0.8778200654676123, gamma=7.625731346670644, learning_rate=0.23211298679519415, max_depth=13, min_child_weight=5.7894051446014565, n_estimators=37, reg_alpha=23.91317623249975, subsample=0.9111970919398438, total=   0.0s
[CV] colsample_bytree=0.8778200654676123, gamma=7.625731346670644, learning_rate=0.23211298679519415, max_depth=13, min_child_weight=5.7894051446014565, n_estimators=37, reg_alpha=23.91317623249975, subsample=0.9111970919398438 
[CV]  colsample_bytree=0.8778200654676123, gamma=7.625731346670644, learning_rate=0.23211298679519415, max_depth=13, min_child_weight=5.7894051446014565, n_estimators=37, reg_alpha=23.91317623249975, subsample=0.9111970919398438, total=   0.0s
[CV] colsample_bytree=0.8778200654676123, gamma=7.625731346670644, learning_rate=0.23211298679519415, max_depth=13, min_child_weight=5.7894051446014565, n_estimators=37, reg_alpha=23.91317623249975, subsample=0.9111970919398438 
[CV]  colsample_bytree=0.8778200654676123, gamma=7.625731346670644, learning_rate=0.23211298679519415, max_depth=13, min_child_weight=5.7894051446014565, n_estimators=37, reg_alpha=23.91317623249975, subsample=0.9111970919398438, total=   0.0s
[CV] colsample_bytree=0.9326188578145767, gamma=6.297175070215645, learning_rate=0.13406960396593584, max_depth=28, min_child_weight=8.84912238209576, n_estimators=3, reg_alpha=51.007740731755845, subsample=0.829697064288153 
[CV]  colsample_bytree=0.9326188578145767, gamma=6.297175070215645, learning_rate=0.13406960396593584, max_depth=28, min_child_weight=8.84912238209576, n_estimators=3, reg_alpha=51.007740731755845, subsample=0.829697064288153, total=   0.0s
[CV] colsample_bytree=0.9326188578145767, gamma=6.297175070215645, learning_rate=0.13406960396593584, max_depth=28, min_child_weight=8.84912238209576, n_estimators=3, reg_alpha=51.007740731755845, subsample=0.829697064288153 
[CV]  colsample_bytree=0.9326188578145767, gamma=6.297175070215645, learning_rate=0.13406960396593584, max_depth=28, min_child_weight=8.84912238209576, n_estimators=3, reg_alpha=51.007740731755845, subsample=0.829697064288153, total=   0.0s
[CV] colsample_bytree=0.9326188578145767, gamma=6.297175070215645, learning_rate=0.13406960396593584, max_depth=28, min_child_weight=8.84912238209576, n_estimators=3, reg_alpha=51.007740731755845, subsample=0.829697064288153 
[CV]  colsample_bytree=0.9326188578145767, gamma=6.297175070215645, learning_rate=0.13406960396593584, max_depth=28, min_child_weight=8.84912238209576, n_estimators=3, reg_alpha=51.007740731755845, subsample=0.829697064288153, total=   0.0s
[CV] colsample_bytree=0.8601544015579499, gamma=9.07815852503524, learning_rate=0.42278882767873494, max_depth=6, min_child_weight=27.826974731547615, n_estimators=19, reg_alpha=47.95708774197891, subsample=0.7572043377602874 
[CV]  colsample_bytree=0.8601544015579499, gamma=9.07815852503524, learning_rate=0.42278882767873494, max_depth=6, min_child_weight=27.826974731547615, n_estimators=19, reg_alpha=47.95708774197891, subsample=0.7572043377602874, total=   0.0s
[CV] colsample_bytree=0.8601544015579499, gamma=9.07815852503524, learning_rate=0.42278882767873494, max_depth=6, min_child_weight=27.826974731547615, n_estimators=19, reg_alpha=47.95708774197891, subsample=0.7572043377602874 
[CV]  colsample_bytree=0.8601544015579499, gamma=9.07815852503524, learning_rate=0.42278882767873494, max_depth=6, min_child_weight=27.826974731547615, n_estimators=19, reg_alpha=47.95708774197891, subsample=0.7572043377602874, total=   0.0s
[CV] colsample_bytree=0.8601544015579499, gamma=9.07815852503524, learning_rate=0.42278882767873494, max_depth=6, min_child_weight=27.826974731547615, n_estimators=19, reg_alpha=47.95708774197891, subsample=0.7572043377602874 
[CV]  colsample_bytree=0.8601544015579499, gamma=9.07815852503524, learning_rate=0.42278882767873494, max_depth=6, min_child_weight=27.826974731547615, n_estimators=19, reg_alpha=47.95708774197891, subsample=0.7572043377602874, total=   0.0s
[CV] colsample_bytree=0.9619178716928831, gamma=4.859906670969098, learning_rate=0.2917241931679893, max_depth=25, min_child_weight=130.30724541975977, n_estimators=15, reg_alpha=22.973117534909058, subsample=0.9964736672519146 
[CV]  colsample_bytree=0.9619178716928831, gamma=4.859906670969098, learning_rate=0.2917241931679893, max_depth=25, min_child_weight=130.30724541975977, n_estimators=15, reg_alpha=22.973117534909058, subsample=0.9964736672519146, total=   0.0s
[CV] colsample_bytree=0.9619178716928831, gamma=4.859906670969098, learning_rate=0.2917241931679893, max_depth=25, min_child_weight=130.30724541975977, n_estimators=15, reg_alpha=22.973117534909058, subsample=0.9964736672519146 
[CV]  colsample_bytree=0.9619178716928831, gamma=4.859906670969098, learning_rate=0.2917241931679893, max_depth=25, min_child_weight=130.30724541975977, n_estimators=15, reg_alpha=22.973117534909058, subsample=0.9964736672519146, total=   0.0s
[CV] colsample_bytree=0.9619178716928831, gamma=4.859906670969098, learning_rate=0.2917241931679893, max_depth=25, min_child_weight=130.30724541975977, n_estimators=15, reg_alpha=22.973117534909058, subsample=0.9964736672519146 
[CV]  colsample_bytree=0.9619178716928831, gamma=4.859906670969098, learning_rate=0.2917241931679893, max_depth=25, min_child_weight=130.30724541975977, n_estimators=15, reg_alpha=22.973117534909058, subsample=0.9964736672519146, total=   0.0s
[CV] colsample_bytree=0.9983645395087906, gamma=4.595452841220066, learning_rate=0.43452690490220625, max_depth=36, min_child_weight=0.7566453892334835, n_estimators=8, reg_alpha=20.178114059119896, subsample=0.8443765778231275 
[CV]  colsample_bytree=0.9983645395087906, gamma=4.595452841220066, learning_rate=0.43452690490220625, max_depth=36, min_child_weight=0.7566453892334835, n_estimators=8, reg_alpha=20.178114059119896, subsample=0.8443765778231275, total=   0.0s
[CV] colsample_bytree=0.9983645395087906, gamma=4.595452841220066, learning_rate=0.43452690490220625, max_depth=36, min_child_weight=0.7566453892334835, n_estimators=8, reg_alpha=20.178114059119896, subsample=0.8443765778231275 
[CV]  colsample_bytree=0.9983645395087906, gamma=4.595452841220066, learning_rate=0.43452690490220625, max_depth=36, min_child_weight=0.7566453892334835, n_estimators=8, reg_alpha=20.178114059119896, subsample=0.8443765778231275, total=   0.0s
[CV] colsample_bytree=0.9983645395087906, gamma=4.595452841220066, learning_rate=0.43452690490220625, max_depth=36, min_child_weight=0.7566453892334835, n_estimators=8, reg_alpha=20.178114059119896, subsample=0.8443765778231275 
[CV]  colsample_bytree=0.9983645395087906, gamma=4.595452841220066, learning_rate=0.43452690490220625, max_depth=36, min_child_weight=0.7566453892334835, n_estimators=8, reg_alpha=20.178114059119896, subsample=0.8443765778231275, total=   0.0s
[CV] colsample_bytree=0.8424955340723784, gamma=8.447344453922218, learning_rate=0.4121569274834597, max_depth=6, min_child_weight=28.103130129138815, n_estimators=22, reg_alpha=16.82393145347256, subsample=0.909026787249163 
[CV]  colsample_bytree=0.8424955340723784, gamma=8.447344453922218, learning_rate=0.4121569274834597, max_depth=6, min_child_weight=28.103130129138815, n_estimators=22, reg_alpha=16.82393145347256, subsample=0.909026787249163, total=   0.0s
[CV] colsample_bytree=0.8424955340723784, gamma=8.447344453922218, learning_rate=0.4121569274834597, max_depth=6, min_child_weight=28.103130129138815, n_estimators=22, reg_alpha=16.82393145347256, subsample=0.909026787249163 
[CV]  colsample_bytree=0.8424955340723784, gamma=8.447344453922218, learning_rate=0.4121569274834597, max_depth=6, min_child_weight=28.103130129138815, n_estimators=22, reg_alpha=16.82393145347256, subsample=0.909026787249163, total=   0.0s
[CV] colsample_bytree=0.8424955340723784, gamma=8.447344453922218, learning_rate=0.4121569274834597, max_depth=6, min_child_weight=28.103130129138815, n_estimators=22, reg_alpha=16.82393145347256, subsample=0.909026787249163 
[CV]  colsample_bytree=0.8424955340723784, gamma=8.447344453922218, learning_rate=0.4121569274834597, max_depth=6, min_child_weight=28.103130129138815, n_estimators=22, reg_alpha=16.82393145347256, subsample=0.909026787249163, total=   0.0s
[CV] colsample_bytree=0.8561630272635711, gamma=3.1524480309537295, learning_rate=0.4071554834100605, max_depth=27, min_child_weight=39.25185668087049, n_estimators=32, reg_alpha=77.54176286499845, subsample=0.9938666866647614 
[CV]  colsample_bytree=0.8561630272635711, gamma=3.1524480309537295, learning_rate=0.4071554834100605, max_depth=27, min_child_weight=39.25185668087049, n_estimators=32, reg_alpha=77.54176286499845, subsample=0.9938666866647614, total=   0.0s
[CV] colsample_bytree=0.8561630272635711, gamma=3.1524480309537295, learning_rate=0.4071554834100605, max_depth=27, min_child_weight=39.25185668087049, n_estimators=32, reg_alpha=77.54176286499845, subsample=0.9938666866647614 
[CV]  colsample_bytree=0.8561630272635711, gamma=3.1524480309537295, learning_rate=0.4071554834100605, max_depth=27, min_child_weight=39.25185668087049, n_estimators=32, reg_alpha=77.54176286499845, subsample=0.9938666866647614, total=   0.0s
[CV] colsample_bytree=0.8561630272635711, gamma=3.1524480309537295, learning_rate=0.4071554834100605, max_depth=27, min_child_weight=39.25185668087049, n_estimators=32, reg_alpha=77.54176286499845, subsample=0.9938666866647614 
[CV]  colsample_bytree=0.8561630272635711, gamma=3.1524480309537295, learning_rate=0.4071554834100605, max_depth=27, min_child_weight=39.25185668087049, n_estimators=32, reg_alpha=77.54176286499845, subsample=0.9938666866647614, total=   0.0s
[CV] colsample_bytree=0.9999752854853113, gamma=9.767591490310597, learning_rate=0.2006321258983098, max_depth=24, min_child_weight=4.2917801169367396, n_estimators=19, reg_alpha=42.749376394757945, subsample=0.9566354330390628 
[CV]  colsample_bytree=0.9999752854853113, gamma=9.767591490310597, learning_rate=0.2006321258983098, max_depth=24, min_child_weight=4.2917801169367396, n_estimators=19, reg_alpha=42.749376394757945, subsample=0.9566354330390628, total=   0.0s
[CV] colsample_bytree=0.9999752854853113, gamma=9.767591490310597, learning_rate=0.2006321258983098, max_depth=24, min_child_weight=4.2917801169367396, n_estimators=19, reg_alpha=42.749376394757945, subsample=0.9566354330390628 
[CV]  colsample_bytree=0.9999752854853113, gamma=9.767591490310597, learning_rate=0.2006321258983098, max_depth=24, min_child_weight=4.2917801169367396, n_estimators=19, reg_alpha=42.749376394757945, subsample=0.9566354330390628, total=   0.0s
[CV] colsample_bytree=0.9999752854853113, gamma=9.767591490310597, learning_rate=0.2006321258983098, max_depth=24, min_child_weight=4.2917801169367396, n_estimators=19, reg_alpha=42.749376394757945, subsample=0.9566354330390628 
[CV]  colsample_bytree=0.9999752854853113, gamma=9.767591490310597, learning_rate=0.2006321258983098, max_depth=24, min_child_weight=4.2917801169367396, n_estimators=19, reg_alpha=42.749376394757945, subsample=0.9566354330390628, total=   0.0s
[CV] colsample_bytree=0.9150263918333182, gamma=6.98057248447303, learning_rate=0.39579177202183996, max_depth=28, min_child_weight=15.340071575524005, n_estimators=5, reg_alpha=24.071655038562053, subsample=0.9568662605812788 
[CV]  colsample_bytree=0.9150263918333182, gamma=6.98057248447303, learning_rate=0.39579177202183996, max_depth=28, min_child_weight=15.340071575524005, n_estimators=5, reg_alpha=24.071655038562053, subsample=0.9568662605812788, total=   0.0s
[CV] colsample_bytree=0.9150263918333182, gamma=6.98057248447303, learning_rate=0.39579177202183996, max_depth=28, min_child_weight=15.340071575524005, n_estimators=5, reg_alpha=24.071655038562053, subsample=0.9568662605812788 
[CV]  colsample_bytree=0.9150263918333182, gamma=6.98057248447303, learning_rate=0.39579177202183996, max_depth=28, min_child_weight=15.340071575524005, n_estimators=5, reg_alpha=24.071655038562053, subsample=0.9568662605812788, total=   0.0s
[CV] colsample_bytree=0.9150263918333182, gamma=6.98057248447303, learning_rate=0.39579177202183996, max_depth=28, min_child_weight=15.340071575524005, n_estimators=5, reg_alpha=24.071655038562053, subsample=0.9568662605812788 
[CV]  colsample_bytree=0.9150263918333182, gamma=6.98057248447303, learning_rate=0.39579177202183996, max_depth=28, min_child_weight=15.340071575524005, n_estimators=5, reg_alpha=24.071655038562053, subsample=0.9568662605812788, total=   0.0s
[CV] colsample_bytree=0.7830976270594145, gamma=6.778008914343111, learning_rate=0.12942795537084434, max_depth=14, min_child_weight=0.5025028937115578, n_estimators=35, reg_alpha=79.88505395560864, subsample=0.7085561122327417 
[CV]  colsample_bytree=0.7830976270594145, gamma=6.778008914343111, learning_rate=0.12942795537084434, max_depth=14, min_child_weight=0.5025028937115578, n_estimators=35, reg_alpha=79.88505395560864, subsample=0.7085561122327417, total=   0.0s
[CV] colsample_bytree=0.7830976270594145, gamma=6.778008914343111, learning_rate=0.12942795537084434, max_depth=14, min_child_weight=0.5025028937115578, n_estimators=35, reg_alpha=79.88505395560864, subsample=0.7085561122327417 
[CV]  colsample_bytree=0.7830976270594145, gamma=6.778008914343111, learning_rate=0.12942795537084434, max_depth=14, min_child_weight=0.5025028937115578, n_estimators=35, reg_alpha=79.88505395560864, subsample=0.7085561122327417, total=   0.0s
[CV] colsample_bytree=0.7830976270594145, gamma=6.778008914343111, learning_rate=0.12942795537084434, max_depth=14, min_child_weight=0.5025028937115578, n_estimators=35, reg_alpha=79.88505395560864, subsample=0.7085561122327417 
[CV]  colsample_bytree=0.7830976270594145, gamma=6.778008914343111, learning_rate=0.12942795537084434, max_depth=14, min_child_weight=0.5025028937115578, n_estimators=35, reg_alpha=79.88505395560864, subsample=0.7085561122327417, total=   0.0s
[CV] colsample_bytree=0.7657053206780995, gamma=5.277146463087466, learning_rate=0.37086443360183924, max_depth=21, min_child_weight=4.3621890684654065, n_estimators=33, reg_alpha=42.06588776117605, subsample=0.9679036217617704 
[CV]  colsample_bytree=0.7657053206780995, gamma=5.277146463087466, learning_rate=0.37086443360183924, max_depth=21, min_child_weight=4.3621890684654065, n_estimators=33, reg_alpha=42.06588776117605, subsample=0.9679036217617704, total=   0.0s
[CV] colsample_bytree=0.7657053206780995, gamma=5.277146463087466, learning_rate=0.37086443360183924, max_depth=21, min_child_weight=4.3621890684654065, n_estimators=33, reg_alpha=42.06588776117605, subsample=0.9679036217617704 
[CV]  colsample_bytree=0.7657053206780995, gamma=5.277146463087466, learning_rate=0.37086443360183924, max_depth=21, min_child_weight=4.3621890684654065, n_estimators=33, reg_alpha=42.06588776117605, subsample=0.9679036217617704, total=   0.0s
[CV] colsample_bytree=0.7657053206780995, gamma=5.277146463087466, learning_rate=0.37086443360183924, max_depth=21, min_child_weight=4.3621890684654065, n_estimators=33, reg_alpha=42.06588776117605, subsample=0.9679036217617704 
[CV]  colsample_bytree=0.7657053206780995, gamma=5.277146463087466, learning_rate=0.37086443360183924, max_depth=21, min_child_weight=4.3621890684654065, n_estimators=33, reg_alpha=42.06588776117605, subsample=0.9679036217617704, total=   0.0s
[CV] colsample_bytree=0.6917736908605385, gamma=1.8161285133076377, learning_rate=0.3747434790882159, max_depth=33, min_child_weight=3.79673720087531, n_estimators=31, reg_alpha=42.13974883372369, subsample=0.9256299657300698 
[CV]  colsample_bytree=0.6917736908605385, gamma=1.8161285133076377, learning_rate=0.3747434790882159, max_depth=33, min_child_weight=3.79673720087531, n_estimators=31, reg_alpha=42.13974883372369, subsample=0.9256299657300698, total=   0.0s
[CV] colsample_bytree=0.6917736908605385, gamma=1.8161285133076377, learning_rate=0.3747434790882159, max_depth=33, min_child_weight=3.79673720087531, n_estimators=31, reg_alpha=42.13974883372369, subsample=0.9256299657300698 
[CV]  colsample_bytree=0.6917736908605385, gamma=1.8161285133076377, learning_rate=0.3747434790882159, max_depth=33, min_child_weight=3.79673720087531, n_estimators=31, reg_alpha=42.13974883372369, subsample=0.9256299657300698, total=   0.0s
[CV] colsample_bytree=0.6917736908605385, gamma=1.8161285133076377, learning_rate=0.3747434790882159, max_depth=33, min_child_weight=3.79673720087531, n_estimators=31, reg_alpha=42.13974883372369, subsample=0.9256299657300698 
[CV]  colsample_bytree=0.6917736908605385, gamma=1.8161285133076377, learning_rate=0.3747434790882159, max_depth=33, min_child_weight=3.79673720087531, n_estimators=31, reg_alpha=42.13974883372369, subsample=0.9256299657300698, total=   0.0s
[CV] colsample_bytree=0.9450387383654071, gamma=9.163055534683508, learning_rate=0.41385420998062283, max_depth=17, min_child_weight=5.876788978186174, n_estimators=20, reg_alpha=34.61579337770725, subsample=0.970726466955887 
[CV]  colsample_bytree=0.9450387383654071, gamma=9.163055534683508, learning_rate=0.41385420998062283, max_depth=17, min_child_weight=5.876788978186174, n_estimators=20, reg_alpha=34.61579337770725, subsample=0.970726466955887, total=   0.0s
[CV] colsample_bytree=0.9450387383654071, gamma=9.163055534683508, learning_rate=0.41385420998062283, max_depth=17, min_child_weight=5.876788978186174, n_estimators=20, reg_alpha=34.61579337770725, subsample=0.970726466955887 
[CV]  colsample_bytree=0.9450387383654071, gamma=9.163055534683508, learning_rate=0.41385420998062283, max_depth=17, min_child_weight=5.876788978186174, n_estimators=20, reg_alpha=34.61579337770725, subsample=0.970726466955887, total=   0.0s
[CV] colsample_bytree=0.9450387383654071, gamma=9.163055534683508, learning_rate=0.41385420998062283, max_depth=17, min_child_weight=5.876788978186174, n_estimators=20, reg_alpha=34.61579337770725, subsample=0.970726466955887 
[CV]  colsample_bytree=0.9450387383654071, gamma=9.163055534683508, learning_rate=0.41385420998062283, max_depth=17, min_child_weight=5.876788978186174, n_estimators=20, reg_alpha=34.61579337770725, subsample=0.970726466955887, total=   0.0s
[CV] colsample_bytree=0.924371319778363, gamma=2.3366613923925006, learning_rate=0.09078690377031898, max_depth=5, min_child_weight=78.12140055629224, n_estimators=35, reg_alpha=8.283349507145983, subsample=0.9140685768819685 
[CV]  colsample_bytree=0.924371319778363, gamma=2.3366613923925006, learning_rate=0.09078690377031898, max_depth=5, min_child_weight=78.12140055629224, n_estimators=35, reg_alpha=8.283349507145983, subsample=0.9140685768819685, total=   0.0s
[CV] colsample_bytree=0.924371319778363, gamma=2.3366613923925006, learning_rate=0.09078690377031898, max_depth=5, min_child_weight=78.12140055629224, n_estimators=35, reg_alpha=8.283349507145983, subsample=0.9140685768819685 
[CV]  colsample_bytree=0.924371319778363, gamma=2.3366613923925006, learning_rate=0.09078690377031898, max_depth=5, min_child_weight=78.12140055629224, n_estimators=35, reg_alpha=8.283349507145983, subsample=0.9140685768819685, total=   0.0s
[CV] colsample_bytree=0.924371319778363, gamma=2.3366613923925006, learning_rate=0.09078690377031898, max_depth=5, min_child_weight=78.12140055629224, n_estimators=35, reg_alpha=8.283349507145983, subsample=0.9140685768819685 
[CV]  colsample_bytree=0.924371319778363, gamma=2.3366613923925006, learning_rate=0.09078690377031898, max_depth=5, min_child_weight=78.12140055629224, n_estimators=35, reg_alpha=8.283349507145983, subsample=0.9140685768819685, total=   0.0s
[CV] colsample_bytree=0.9656665471056349, gamma=5.19351824366033, learning_rate=0.3641184112886476, max_depth=38, min_child_weight=18.576946077130895, n_estimators=18, reg_alpha=4.211692680168841, subsample=0.9999075984465978 
[CV]  colsample_bytree=0.9656665471056349, gamma=5.19351824366033, learning_rate=0.3641184112886476, max_depth=38, min_child_weight=18.576946077130895, n_estimators=18, reg_alpha=4.211692680168841, subsample=0.9999075984465978, total=   0.0s
[CV] colsample_bytree=0.9656665471056349, gamma=5.19351824366033, learning_rate=0.3641184112886476, max_depth=38, min_child_weight=18.576946077130895, n_estimators=18, reg_alpha=4.211692680168841, subsample=0.9999075984465978 
[CV]  colsample_bytree=0.9656665471056349, gamma=5.19351824366033, learning_rate=0.3641184112886476, max_depth=38, min_child_weight=18.576946077130895, n_estimators=18, reg_alpha=4.211692680168841, subsample=0.9999075984465978, total=   0.0s
[CV] colsample_bytree=0.9656665471056349, gamma=5.19351824366033, learning_rate=0.3641184112886476, max_depth=38, min_child_weight=18.576946077130895, n_estimators=18, reg_alpha=4.211692680168841, subsample=0.9999075984465978 
[CV]  colsample_bytree=0.9656665471056349, gamma=5.19351824366033, learning_rate=0.3641184112886476, max_depth=38, min_child_weight=18.576946077130895, n_estimators=18, reg_alpha=4.211692680168841, subsample=0.9999075984465978, total=   0.0s
[CV] colsample_bytree=0.9425522100412647, gamma=0.47318232441725727, learning_rate=0.3393459013209253, max_depth=20, min_child_weight=47.11130994476061, n_estimators=30, reg_alpha=44.017135477067775, subsample=0.968999053589992 
[CV]  colsample_bytree=0.9425522100412647, gamma=0.47318232441725727, learning_rate=0.3393459013209253, max_depth=20, min_child_weight=47.11130994476061, n_estimators=30, reg_alpha=44.017135477067775, subsample=0.968999053589992, total=   0.0s
[CV] colsample_bytree=0.9425522100412647, gamma=0.47318232441725727, learning_rate=0.3393459013209253, max_depth=20, min_child_weight=47.11130994476061, n_estimators=30, reg_alpha=44.017135477067775, subsample=0.968999053589992 
[CV]  colsample_bytree=0.9425522100412647, gamma=0.47318232441725727, learning_rate=0.3393459013209253, max_depth=20, min_child_weight=47.11130994476061, n_estimators=30, reg_alpha=44.017135477067775, subsample=0.968999053589992, total=   0.0s
[CV] colsample_bytree=0.9425522100412647, gamma=0.47318232441725727, learning_rate=0.3393459013209253, max_depth=20, min_child_weight=47.11130994476061, n_estimators=30, reg_alpha=44.017135477067775, subsample=0.968999053589992 
[CV]  colsample_bytree=0.9425522100412647, gamma=0.47318232441725727, learning_rate=0.3393459013209253, max_depth=20, min_child_weight=47.11130994476061, n_estimators=30, reg_alpha=44.017135477067775, subsample=0.968999053589992, total=   0.0s
[CV] colsample_bytree=0.8167406971208426, gamma=5.126616988043951, learning_rate=0.29610597426518626, max_depth=6, min_child_weight=232.6748868841225, n_estimators=36, reg_alpha=7.716208058769659, subsample=0.9067978731045314 
[CV]  colsample_bytree=0.8167406971208426, gamma=5.126616988043951, learning_rate=0.29610597426518626, max_depth=6, min_child_weight=232.6748868841225, n_estimators=36, reg_alpha=7.716208058769659, subsample=0.9067978731045314, total=   0.0s
[CV] colsample_bytree=0.8167406971208426, gamma=5.126616988043951, learning_rate=0.29610597426518626, max_depth=6, min_child_weight=232.6748868841225, n_estimators=36, reg_alpha=7.716208058769659, subsample=0.9067978731045314 
[CV]  colsample_bytree=0.8167406971208426, gamma=5.126616988043951, learning_rate=0.29610597426518626, max_depth=6, min_child_weight=232.6748868841225, n_estimators=36, reg_alpha=7.716208058769659, subsample=0.9067978731045314, total=   0.0s
[CV] colsample_bytree=0.8167406971208426, gamma=5.126616988043951, learning_rate=0.29610597426518626, max_depth=6, min_child_weight=232.6748868841225, n_estimators=36, reg_alpha=7.716208058769659, subsample=0.9067978731045314 
[CV]  colsample_bytree=0.8167406971208426, gamma=5.126616988043951, learning_rate=0.29610597426518626, max_depth=6, min_child_weight=232.6748868841225, n_estimators=36, reg_alpha=7.716208058769659, subsample=0.9067978731045314, total=   0.0s
[CV] colsample_bytree=0.9258153285652542, gamma=5.264259339055213, learning_rate=0.10417116122886796, max_depth=32, min_child_weight=1.3284201131548217, n_estimators=32, reg_alpha=43.05219742707525, subsample=0.964807626324511 
[CV]  colsample_bytree=0.9258153285652542, gamma=5.264259339055213, learning_rate=0.10417116122886796, max_depth=32, min_child_weight=1.3284201131548217, n_estimators=32, reg_alpha=43.05219742707525, subsample=0.964807626324511, total=   0.0s
[CV] colsample_bytree=0.9258153285652542, gamma=5.264259339055213, learning_rate=0.10417116122886796, max_depth=32, min_child_weight=1.3284201131548217, n_estimators=32, reg_alpha=43.05219742707525, subsample=0.964807626324511 
[CV]  colsample_bytree=0.9258153285652542, gamma=5.264259339055213, learning_rate=0.10417116122886796, max_depth=32, min_child_weight=1.3284201131548217, n_estimators=32, reg_alpha=43.05219742707525, subsample=0.964807626324511, total=   0.0s
[CV] colsample_bytree=0.9258153285652542, gamma=5.264259339055213, learning_rate=0.10417116122886796, max_depth=32, min_child_weight=1.3284201131548217, n_estimators=32, reg_alpha=43.05219742707525, subsample=0.964807626324511 
[CV]  colsample_bytree=0.9258153285652542, gamma=5.264259339055213, learning_rate=0.10417116122886796, max_depth=32, min_child_weight=1.3284201131548217, n_estimators=32, reg_alpha=43.05219742707525, subsample=0.964807626324511, total=   0.0s
[CV] colsample_bytree=0.9493404422009409, gamma=8.414100590490124, learning_rate=0.2722110139774603, max_depth=35, min_child_weight=11.401914629021704, n_estimators=35, reg_alpha=52.02951670166972, subsample=0.9812982557044617 
[CV]  colsample_bytree=0.9493404422009409, gamma=8.414100590490124, learning_rate=0.2722110139774603, max_depth=35, min_child_weight=11.401914629021704, n_estimators=35, reg_alpha=52.02951670166972, subsample=0.9812982557044617, total=   0.0s
[CV] colsample_bytree=0.9493404422009409, gamma=8.414100590490124, learning_rate=0.2722110139774603, max_depth=35, min_child_weight=11.401914629021704, n_estimators=35, reg_alpha=52.02951670166972, subsample=0.9812982557044617 
[CV]  colsample_bytree=0.9493404422009409, gamma=8.414100590490124, learning_rate=0.2722110139774603, max_depth=35, min_child_weight=11.401914629021704, n_estimators=35, reg_alpha=52.02951670166972, subsample=0.9812982557044617, total=   0.0s
[CV] colsample_bytree=0.9493404422009409, gamma=8.414100590490124, learning_rate=0.2722110139774603, max_depth=35, min_child_weight=11.401914629021704, n_estimators=35, reg_alpha=52.02951670166972, subsample=0.9812982557044617 
[CV]  colsample_bytree=0.9493404422009409, gamma=8.414100590490124, learning_rate=0.2722110139774603, max_depth=35, min_child_weight=11.401914629021704, n_estimators=35, reg_alpha=52.02951670166972, subsample=0.9812982557044617, total=   0.0s
[CV] colsample_bytree=0.962959330606662, gamma=4.890072943836898, learning_rate=0.13146973131129874, max_depth=34, min_child_weight=92.84384515314294, n_estimators=22, reg_alpha=34.957913776821954, subsample=0.8500481920610294 
[CV]  colsample_bytree=0.962959330606662, gamma=4.890072943836898, learning_rate=0.13146973131129874, max_depth=34, min_child_weight=92.84384515314294, n_estimators=22, reg_alpha=34.957913776821954, subsample=0.8500481920610294, total=   0.0s
[CV] colsample_bytree=0.962959330606662, gamma=4.890072943836898, learning_rate=0.13146973131129874, max_depth=34, min_child_weight=92.84384515314294, n_estimators=22, reg_alpha=34.957913776821954, subsample=0.8500481920610294 
[CV]  colsample_bytree=0.962959330606662, gamma=4.890072943836898, learning_rate=0.13146973131129874, max_depth=34, min_child_weight=92.84384515314294, n_estimators=22, reg_alpha=34.957913776821954, subsample=0.8500481920610294, total=   0.0s
[CV] colsample_bytree=0.962959330606662, gamma=4.890072943836898, learning_rate=0.13146973131129874, max_depth=34, min_child_weight=92.84384515314294, n_estimators=22, reg_alpha=34.957913776821954, subsample=0.8500481920610294 
[CV]  colsample_bytree=0.962959330606662, gamma=4.890072943836898, learning_rate=0.13146973131129874, max_depth=34, min_child_weight=92.84384515314294, n_estimators=22, reg_alpha=34.957913776821954, subsample=0.8500481920610294, total=   0.0s
[CV] colsample_bytree=0.9356221125253228, gamma=1.3242963302881627, learning_rate=0.19883507176503223, max_depth=25, min_child_weight=35.23328526088029, n_estimators=4, reg_alpha=4.2492514733560665, subsample=0.984374411206703 
[CV]  colsample_bytree=0.9356221125253228, gamma=1.3242963302881627, learning_rate=0.19883507176503223, max_depth=25, min_child_weight=35.23328526088029, n_estimators=4, reg_alpha=4.2492514733560665, subsample=0.984374411206703, total=   0.0s
[CV] colsample_bytree=0.9356221125253228, gamma=1.3242963302881627, learning_rate=0.19883507176503223, max_depth=25, min_child_weight=35.23328526088029, n_estimators=4, reg_alpha=4.2492514733560665, subsample=0.984374411206703 
[CV]  colsample_bytree=0.9356221125253228, gamma=1.3242963302881627, learning_rate=0.19883507176503223, max_depth=25, min_child_weight=35.23328526088029, n_estimators=4, reg_alpha=4.2492514733560665, subsample=0.984374411206703, total=   0.0s
[CV] colsample_bytree=0.9356221125253228, gamma=1.3242963302881627, learning_rate=0.19883507176503223, max_depth=25, min_child_weight=35.23328526088029, n_estimators=4, reg_alpha=4.2492514733560665, subsample=0.984374411206703 
[CV]  colsample_bytree=0.9356221125253228, gamma=1.3242963302881627, learning_rate=0.19883507176503223, max_depth=25, min_child_weight=35.23328526088029, n_estimators=4, reg_alpha=4.2492514733560665, subsample=0.984374411206703, total=   0.0s
[CV] colsample_bytree=0.9046973777610755, gamma=3.424541198435198, learning_rate=0.38834059321107106, max_depth=27, min_child_weight=85.12384192430562, n_estimators=36, reg_alpha=0.5672209744063974, subsample=0.9247620384982331 
[CV]  colsample_bytree=0.9046973777610755, gamma=3.424541198435198, learning_rate=0.38834059321107106, max_depth=27, min_child_weight=85.12384192430562, n_estimators=36, reg_alpha=0.5672209744063974, subsample=0.9247620384982331, total=   0.0s
[CV] colsample_bytree=0.9046973777610755, gamma=3.424541198435198, learning_rate=0.38834059321107106, max_depth=27, min_child_weight=85.12384192430562, n_estimators=36, reg_alpha=0.5672209744063974, subsample=0.9247620384982331 
[CV]  colsample_bytree=0.9046973777610755, gamma=3.424541198435198, learning_rate=0.38834059321107106, max_depth=27, min_child_weight=85.12384192430562, n_estimators=36, reg_alpha=0.5672209744063974, subsample=0.9247620384982331, total=   0.0s
[CV] colsample_bytree=0.9046973777610755, gamma=3.424541198435198, learning_rate=0.38834059321107106, max_depth=27, min_child_weight=85.12384192430562, n_estimators=36, reg_alpha=0.5672209744063974, subsample=0.9247620384982331 
[CV]  colsample_bytree=0.9046973777610755, gamma=3.424541198435198, learning_rate=0.38834059321107106, max_depth=27, min_child_weight=85.12384192430562, n_estimators=36, reg_alpha=0.5672209744063974, subsample=0.9247620384982331, total=   0.0s
[CV] colsample_bytree=0.9651894037352509, gamma=9.586999025558045, learning_rate=0.4447771262969091, max_depth=21, min_child_weight=30.11094999860071, n_estimators=14, reg_alpha=44.01638412606923, subsample=0.9944472853908971 
[CV]  colsample_bytree=0.9651894037352509, gamma=9.586999025558045, learning_rate=0.4447771262969091, max_depth=21, min_child_weight=30.11094999860071, n_estimators=14, reg_alpha=44.01638412606923, subsample=0.9944472853908971, total=   0.0s
[CV] colsample_bytree=0.9651894037352509, gamma=9.586999025558045, learning_rate=0.4447771262969091, max_depth=21, min_child_weight=30.11094999860071, n_estimators=14, reg_alpha=44.01638412606923, subsample=0.9944472853908971 
[CV]  colsample_bytree=0.9651894037352509, gamma=9.586999025558045, learning_rate=0.4447771262969091, max_depth=21, min_child_weight=30.11094999860071, n_estimators=14, reg_alpha=44.01638412606923, subsample=0.9944472853908971, total=   0.0s
[CV] colsample_bytree=0.9651894037352509, gamma=9.586999025558045, learning_rate=0.4447771262969091, max_depth=21, min_child_weight=30.11094999860071, n_estimators=14, reg_alpha=44.01638412606923, subsample=0.9944472853908971 
[CV]  colsample_bytree=0.9651894037352509, gamma=9.586999025558045, learning_rate=0.4447771262969091, max_depth=21, min_child_weight=30.11094999860071, n_estimators=14, reg_alpha=44.01638412606923, subsample=0.9944472853908971, total=   0.0s
[CV] colsample_bytree=0.9749001820628698, gamma=7.762146287068875, learning_rate=0.4060665213941543, max_depth=31, min_child_weight=123.41682672745611, n_estimators=36, reg_alpha=47.76512363859637, subsample=0.9284848200851272 
[CV]  colsample_bytree=0.9749001820628698, gamma=7.762146287068875, learning_rate=0.4060665213941543, max_depth=31, min_child_weight=123.41682672745611, n_estimators=36, reg_alpha=47.76512363859637, subsample=0.9284848200851272, total=   0.0s
[CV] colsample_bytree=0.9749001820628698, gamma=7.762146287068875, learning_rate=0.4060665213941543, max_depth=31, min_child_weight=123.41682672745611, n_estimators=36, reg_alpha=47.76512363859637, subsample=0.9284848200851272 
[CV]  colsample_bytree=0.9749001820628698, gamma=7.762146287068875, learning_rate=0.4060665213941543, max_depth=31, min_child_weight=123.41682672745611, n_estimators=36, reg_alpha=47.76512363859637, subsample=0.9284848200851272, total=   0.0s
[CV] colsample_bytree=0.9749001820628698, gamma=7.762146287068875, learning_rate=0.4060665213941543, max_depth=31, min_child_weight=123.41682672745611, n_estimators=36, reg_alpha=47.76512363859637, subsample=0.9284848200851272 
[CV]  colsample_bytree=0.9749001820628698, gamma=7.762146287068875, learning_rate=0.4060665213941543, max_depth=31, min_child_weight=123.41682672745611, n_estimators=36, reg_alpha=47.76512363859637, subsample=0.9284848200851272, total=   0.0s
[CV] colsample_bytree=0.9778015162224186, gamma=9.997691169446238, learning_rate=0.2794927604492029, max_depth=38, min_child_weight=33.637785648614084, n_estimators=5, reg_alpha=83.7283205207771, subsample=0.97079092568387 
[CV]  colsample_bytree=0.9778015162224186, gamma=9.997691169446238, learning_rate=0.2794927604492029, max_depth=38, min_child_weight=33.637785648614084, n_estimators=5, reg_alpha=83.7283205207771, subsample=0.97079092568387, total=   0.0s
[CV] colsample_bytree=0.9778015162224186, gamma=9.997691169446238, learning_rate=0.2794927604492029, max_depth=38, min_child_weight=33.637785648614084, n_estimators=5, reg_alpha=83.7283205207771, subsample=0.97079092568387 
[CV]  colsample_bytree=0.9778015162224186, gamma=9.997691169446238, learning_rate=0.2794927604492029, max_depth=38, min_child_weight=33.637785648614084, n_estimators=5, reg_alpha=83.7283205207771, subsample=0.97079092568387, total=   0.0s
[CV] colsample_bytree=0.9778015162224186, gamma=9.997691169446238, learning_rate=0.2794927604492029, max_depth=38, min_child_weight=33.637785648614084, n_estimators=5, reg_alpha=83.7283205207771, subsample=0.97079092568387 
[CV]  colsample_bytree=0.9778015162224186, gamma=9.997691169446238, learning_rate=0.2794927604492029, max_depth=38, min_child_weight=33.637785648614084, n_estimators=5, reg_alpha=83.7283205207771, subsample=0.97079092568387, total=   0.0s
[CV] colsample_bytree=0.962838002424029, gamma=5.336533449712114, learning_rate=0.17578680447433986, max_depth=27, min_child_weight=6.635445874743752, n_estimators=30, reg_alpha=28.42205951066925, subsample=0.8218092655515985 
[CV]  colsample_bytree=0.962838002424029, gamma=5.336533449712114, learning_rate=0.17578680447433986, max_depth=27, min_child_weight=6.635445874743752, n_estimators=30, reg_alpha=28.42205951066925, subsample=0.8218092655515985, total=   0.0s
[CV] colsample_bytree=0.962838002424029, gamma=5.336533449712114, learning_rate=0.17578680447433986, max_depth=27, min_child_weight=6.635445874743752, n_estimators=30, reg_alpha=28.42205951066925, subsample=0.8218092655515985 
[CV]  colsample_bytree=0.962838002424029, gamma=5.336533449712114, learning_rate=0.17578680447433986, max_depth=27, min_child_weight=6.635445874743752, n_estimators=30, reg_alpha=28.42205951066925, subsample=0.8218092655515985, total=   0.0s
[CV] colsample_bytree=0.962838002424029, gamma=5.336533449712114, learning_rate=0.17578680447433986, max_depth=27, min_child_weight=6.635445874743752, n_estimators=30, reg_alpha=28.42205951066925, subsample=0.8218092655515985 
[CV]  colsample_bytree=0.962838002424029, gamma=5.336533449712114, learning_rate=0.17578680447433986, max_depth=27, min_child_weight=6.635445874743752, n_estimators=30, reg_alpha=28.42205951066925, subsample=0.8218092655515985, total=   0.0s
[CV] colsample_bytree=0.8498504401772669, gamma=6.902278681627479, learning_rate=0.231141399980615, max_depth=6, min_child_weight=17.131336453946968, n_estimators=32, reg_alpha=112.5454845103544, subsample=0.8596416843808513 
[CV]  colsample_bytree=0.8498504401772669, gamma=6.902278681627479, learning_rate=0.231141399980615, max_depth=6, min_child_weight=17.131336453946968, n_estimators=32, reg_alpha=112.5454845103544, subsample=0.8596416843808513, total=   0.0s
[CV] colsample_bytree=0.8498504401772669, gamma=6.902278681627479, learning_rate=0.231141399980615, max_depth=6, min_child_weight=17.131336453946968, n_estimators=32, reg_alpha=112.5454845103544, subsample=0.8596416843808513 
[CV]  colsample_bytree=0.8498504401772669, gamma=6.902278681627479, learning_rate=0.231141399980615, max_depth=6, min_child_weight=17.131336453946968, n_estimators=32, reg_alpha=112.5454845103544, subsample=0.8596416843808513, total=   0.0s
[CV] colsample_bytree=0.8498504401772669, gamma=6.902278681627479, learning_rate=0.231141399980615, max_depth=6, min_child_weight=17.131336453946968, n_estimators=32, reg_alpha=112.5454845103544, subsample=0.8596416843808513 
[CV]  colsample_bytree=0.8498504401772669, gamma=6.902278681627479, learning_rate=0.231141399980615, max_depth=6, min_child_weight=17.131336453946968, n_estimators=32, reg_alpha=112.5454845103544, subsample=0.8596416843808513, total=   0.0s
[CV] colsample_bytree=0.7694156747825355, gamma=0.5944907614485251, learning_rate=0.40459979455729833, max_depth=23, min_child_weight=93.00082832642657, n_estimators=7, reg_alpha=22.034676954560886, subsample=0.8786947404965245 
[CV]  colsample_bytree=0.7694156747825355, gamma=0.5944907614485251, learning_rate=0.40459979455729833, max_depth=23, min_child_weight=93.00082832642657, n_estimators=7, reg_alpha=22.034676954560886, subsample=0.8786947404965245, total=   0.0s
[CV] colsample_bytree=0.7694156747825355, gamma=0.5944907614485251, learning_rate=0.40459979455729833, max_depth=23, min_child_weight=93.00082832642657, n_estimators=7, reg_alpha=22.034676954560886, subsample=0.8786947404965245 
[CV]  colsample_bytree=0.7694156747825355, gamma=0.5944907614485251, learning_rate=0.40459979455729833, max_depth=23, min_child_weight=93.00082832642657, n_estimators=7, reg_alpha=22.034676954560886, subsample=0.8786947404965245, total=   0.0s
[CV] colsample_bytree=0.7694156747825355, gamma=0.5944907614485251, learning_rate=0.40459979455729833, max_depth=23, min_child_weight=93.00082832642657, n_estimators=7, reg_alpha=22.034676954560886, subsample=0.8786947404965245 
[CV]  colsample_bytree=0.7694156747825355, gamma=0.5944907614485251, learning_rate=0.40459979455729833, max_depth=23, min_child_weight=93.00082832642657, n_estimators=7, reg_alpha=22.034676954560886, subsample=0.8786947404965245, total=   0.0s
[CV] colsample_bytree=0.8095793735045576, gamma=4.784503456129699, learning_rate=0.20966135876466158, max_depth=25, min_child_weight=12.991263135531565, n_estimators=34, reg_alpha=7.466469039520882, subsample=0.9699481172402621 
[CV]  colsample_bytree=0.8095793735045576, gamma=4.784503456129699, learning_rate=0.20966135876466158, max_depth=25, min_child_weight=12.991263135531565, n_estimators=34, reg_alpha=7.466469039520882, subsample=0.9699481172402621, total=   0.0s
[CV] colsample_bytree=0.8095793735045576, gamma=4.784503456129699, learning_rate=0.20966135876466158, max_depth=25, min_child_weight=12.991263135531565, n_estimators=34, reg_alpha=7.466469039520882, subsample=0.9699481172402621 
[CV]  colsample_bytree=0.8095793735045576, gamma=4.784503456129699, learning_rate=0.20966135876466158, max_depth=25, min_child_weight=12.991263135531565, n_estimators=34, reg_alpha=7.466469039520882, subsample=0.9699481172402621, total=   0.0s
[CV] colsample_bytree=0.8095793735045576, gamma=4.784503456129699, learning_rate=0.20966135876466158, max_depth=25, min_child_weight=12.991263135531565, n_estimators=34, reg_alpha=7.466469039520882, subsample=0.9699481172402621 
[CV]  colsample_bytree=0.8095793735045576, gamma=4.784503456129699, learning_rate=0.20966135876466158, max_depth=25, min_child_weight=12.991263135531565, n_estimators=34, reg_alpha=7.466469039520882, subsample=0.9699481172402621, total=   0.0s
[CV] colsample_bytree=0.952564311302402, gamma=1.5599250030509426, learning_rate=0.1716843304084616, max_depth=8, min_child_weight=4.912465009470827, n_estimators=30, reg_alpha=105.87004739595281, subsample=0.9159861181947812 
[CV]  colsample_bytree=0.952564311302402, gamma=1.5599250030509426, learning_rate=0.1716843304084616, max_depth=8, min_child_weight=4.912465009470827, n_estimators=30, reg_alpha=105.87004739595281, subsample=0.9159861181947812, total=   0.0s
[CV] colsample_bytree=0.952564311302402, gamma=1.5599250030509426, learning_rate=0.1716843304084616, max_depth=8, min_child_weight=4.912465009470827, n_estimators=30, reg_alpha=105.87004739595281, subsample=0.9159861181947812 
[CV]  colsample_bytree=0.952564311302402, gamma=1.5599250030509426, learning_rate=0.1716843304084616, max_depth=8, min_child_weight=4.912465009470827, n_estimators=30, reg_alpha=105.87004739595281, subsample=0.9159861181947812, total=   0.0s
[CV] colsample_bytree=0.952564311302402, gamma=1.5599250030509426, learning_rate=0.1716843304084616, max_depth=8, min_child_weight=4.912465009470827, n_estimators=30, reg_alpha=105.87004739595281, subsample=0.9159861181947812 
[CV]  colsample_bytree=0.952564311302402, gamma=1.5599250030509426, learning_rate=0.1716843304084616, max_depth=8, min_child_weight=4.912465009470827, n_estimators=30, reg_alpha=105.87004739595281, subsample=0.9159861181947812, total=   0.0s
[CV] colsample_bytree=0.9809643582069478, gamma=6.248070796484706, learning_rate=0.07251746796632506, max_depth=29, min_child_weight=21.770919515440852, n_estimators=38, reg_alpha=44.00785071169654, subsample=0.9234221873035902 
[CV]  colsample_bytree=0.9809643582069478, gamma=6.248070796484706, learning_rate=0.07251746796632506, max_depth=29, min_child_weight=21.770919515440852, n_estimators=38, reg_alpha=44.00785071169654, subsample=0.9234221873035902, total=   0.0s
[CV] colsample_bytree=0.9809643582069478, gamma=6.248070796484706, learning_rate=0.07251746796632506, max_depth=29, min_child_weight=21.770919515440852, n_estimators=38, reg_alpha=44.00785071169654, subsample=0.9234221873035902 
[CV]  colsample_bytree=0.9809643582069478, gamma=6.248070796484706, learning_rate=0.07251746796632506, max_depth=29, min_child_weight=21.770919515440852, n_estimators=38, reg_alpha=44.00785071169654, subsample=0.9234221873035902, total=   0.0s
[CV] colsample_bytree=0.9809643582069478, gamma=6.248070796484706, learning_rate=0.07251746796632506, max_depth=29, min_child_weight=21.770919515440852, n_estimators=38, reg_alpha=44.00785071169654, subsample=0.9234221873035902 
[CV]  colsample_bytree=0.9809643582069478, gamma=6.248070796484706, learning_rate=0.07251746796632506, max_depth=29, min_child_weight=21.770919515440852, n_estimators=38, reg_alpha=44.00785071169654, subsample=0.9234221873035902, total=   0.0s
[CV] colsample_bytree=0.5983491956727909, gamma=1.6577417567546826, learning_rate=0.23516803484770066, max_depth=35, min_child_weight=1.8896609435101328, n_estimators=4, reg_alpha=6.666430265744454, subsample=0.9250726574192653 
[CV]  colsample_bytree=0.5983491956727909, gamma=1.6577417567546826, learning_rate=0.23516803484770066, max_depth=35, min_child_weight=1.8896609435101328, n_estimators=4, reg_alpha=6.666430265744454, subsample=0.9250726574192653, total=   0.0s
[CV] colsample_bytree=0.5983491956727909, gamma=1.6577417567546826, learning_rate=0.23516803484770066, max_depth=35, min_child_weight=1.8896609435101328, n_estimators=4, reg_alpha=6.666430265744454, subsample=0.9250726574192653 
[CV]  colsample_bytree=0.5983491956727909, gamma=1.6577417567546826, learning_rate=0.23516803484770066, max_depth=35, min_child_weight=1.8896609435101328, n_estimators=4, reg_alpha=6.666430265744454, subsample=0.9250726574192653, total=   0.0s
[CV] colsample_bytree=0.5983491956727909, gamma=1.6577417567546826, learning_rate=0.23516803484770066, max_depth=35, min_child_weight=1.8896609435101328, n_estimators=4, reg_alpha=6.666430265744454, subsample=0.9250726574192653 
[CV]  colsample_bytree=0.5983491956727909, gamma=1.6577417567546826, learning_rate=0.23516803484770066, max_depth=35, min_child_weight=1.8896609435101328, n_estimators=4, reg_alpha=6.666430265744454, subsample=0.9250726574192653, total=   0.0s
[CV] colsample_bytree=0.9982956511648968, gamma=5.190823222119895, learning_rate=0.37399548426496654, max_depth=21, min_child_weight=184.42659907017907, n_estimators=27, reg_alpha=58.04697908379749, subsample=0.8306848440514973 
[CV]  colsample_bytree=0.9982956511648968, gamma=5.190823222119895, learning_rate=0.37399548426496654, max_depth=21, min_child_weight=184.42659907017907, n_estimators=27, reg_alpha=58.04697908379749, subsample=0.8306848440514973, total=   0.0s
[CV] colsample_bytree=0.9982956511648968, gamma=5.190823222119895, learning_rate=0.37399548426496654, max_depth=21, min_child_weight=184.42659907017907, n_estimators=27, reg_alpha=58.04697908379749, subsample=0.8306848440514973 
[CV]  colsample_bytree=0.9982956511648968, gamma=5.190823222119895, learning_rate=0.37399548426496654, max_depth=21, min_child_weight=184.42659907017907, n_estimators=27, reg_alpha=58.04697908379749, subsample=0.8306848440514973, total=   0.0s
[CV] colsample_bytree=0.9982956511648968, gamma=5.190823222119895, learning_rate=0.37399548426496654, max_depth=21, min_child_weight=184.42659907017907, n_estimators=27, reg_alpha=58.04697908379749, subsample=0.8306848440514973 
[CV]  colsample_bytree=0.9982956511648968, gamma=5.190823222119895, learning_rate=0.37399548426496654, max_depth=21, min_child_weight=184.42659907017907, n_estimators=27, reg_alpha=58.04697908379749, subsample=0.8306848440514973, total=   0.0s
[CV] colsample_bytree=0.928923551364681, gamma=6.736984238986317, learning_rate=0.28139983191568024, max_depth=35, min_child_weight=81.89478220134201, n_estimators=8, reg_alpha=21.726390605557174, subsample=0.9777018059335505 
[CV]  colsample_bytree=0.928923551364681, gamma=6.736984238986317, learning_rate=0.28139983191568024, max_depth=35, min_child_weight=81.89478220134201, n_estimators=8, reg_alpha=21.726390605557174, subsample=0.9777018059335505, total=   0.0s
[CV] colsample_bytree=0.928923551364681, gamma=6.736984238986317, learning_rate=0.28139983191568024, max_depth=35, min_child_weight=81.89478220134201, n_estimators=8, reg_alpha=21.726390605557174, subsample=0.9777018059335505 
[CV]  colsample_bytree=0.928923551364681, gamma=6.736984238986317, learning_rate=0.28139983191568024, max_depth=35, min_child_weight=81.89478220134201, n_estimators=8, reg_alpha=21.726390605557174, subsample=0.9777018059335505, total=   0.0s
[CV] colsample_bytree=0.928923551364681, gamma=6.736984238986317, learning_rate=0.28139983191568024, max_depth=35, min_child_weight=81.89478220134201, n_estimators=8, reg_alpha=21.726390605557174, subsample=0.9777018059335505 
[CV]  colsample_bytree=0.928923551364681, gamma=6.736984238986317, learning_rate=0.28139983191568024, max_depth=35, min_child_weight=81.89478220134201, n_estimators=8, reg_alpha=21.726390605557174, subsample=0.9777018059335505, total=   0.0s
[CV] colsample_bytree=0.9285499018334513, gamma=8.076984091122094, learning_rate=0.19141394506719617, max_depth=7, min_child_weight=92.76899823664898, n_estimators=26, reg_alpha=187.4972937471552, subsample=0.9866161327035416 
[CV]  colsample_bytree=0.9285499018334513, gamma=8.076984091122094, learning_rate=0.19141394506719617, max_depth=7, min_child_weight=92.76899823664898, n_estimators=26, reg_alpha=187.4972937471552, subsample=0.9866161327035416, total=   0.0s
[CV] colsample_bytree=0.9285499018334513, gamma=8.076984091122094, learning_rate=0.19141394506719617, max_depth=7, min_child_weight=92.76899823664898, n_estimators=26, reg_alpha=187.4972937471552, subsample=0.9866161327035416 
[CV]  colsample_bytree=0.9285499018334513, gamma=8.076984091122094, learning_rate=0.19141394506719617, max_depth=7, min_child_weight=92.76899823664898, n_estimators=26, reg_alpha=187.4972937471552, subsample=0.9866161327035416, total=   0.0s
[CV] colsample_bytree=0.9285499018334513, gamma=8.076984091122094, learning_rate=0.19141394506719617, max_depth=7, min_child_weight=92.76899823664898, n_estimators=26, reg_alpha=187.4972937471552, subsample=0.9866161327035416 
[CV]  colsample_bytree=0.9285499018334513, gamma=8.076984091122094, learning_rate=0.19141394506719617, max_depth=7, min_child_weight=92.76899823664898, n_estimators=26, reg_alpha=187.4972937471552, subsample=0.9866161327035416, total=   0.0s
[CV] colsample_bytree=0.9375905300738867, gamma=7.339429857330612, learning_rate=0.12390216622030842, max_depth=31, min_child_weight=53.94974491014385, n_estimators=15, reg_alpha=9.201330238445792, subsample=0.8689496700512581 
[CV]  colsample_bytree=0.9375905300738867, gamma=7.339429857330612, learning_rate=0.12390216622030842, max_depth=31, min_child_weight=53.94974491014385, n_estimators=15, reg_alpha=9.201330238445792, subsample=0.8689496700512581, total=   0.0s
[CV] colsample_bytree=0.9375905300738867, gamma=7.339429857330612, learning_rate=0.12390216622030842, max_depth=31, min_child_weight=53.94974491014385, n_estimators=15, reg_alpha=9.201330238445792, subsample=0.8689496700512581 
[CV]  colsample_bytree=0.9375905300738867, gamma=7.339429857330612, learning_rate=0.12390216622030842, max_depth=31, min_child_weight=53.94974491014385, n_estimators=15, reg_alpha=9.201330238445792, subsample=0.8689496700512581, total=   0.0s
[CV] colsample_bytree=0.9375905300738867, gamma=7.339429857330612, learning_rate=0.12390216622030842, max_depth=31, min_child_weight=53.94974491014385, n_estimators=15, reg_alpha=9.201330238445792, subsample=0.8689496700512581 
[CV]  colsample_bytree=0.9375905300738867, gamma=7.339429857330612, learning_rate=0.12390216622030842, max_depth=31, min_child_weight=53.94974491014385, n_estimators=15, reg_alpha=9.201330238445792, subsample=0.8689496700512581, total=   0.0s
[CV] colsample_bytree=0.9420989706632965, gamma=4.84743784631088, learning_rate=0.2971593794925696, max_depth=12, min_child_weight=61.21758661968412, n_estimators=21, reg_alpha=3.5194407158563146, subsample=0.9238702912817508 
[CV]  colsample_bytree=0.9420989706632965, gamma=4.84743784631088, learning_rate=0.2971593794925696, max_depth=12, min_child_weight=61.21758661968412, n_estimators=21, reg_alpha=3.5194407158563146, subsample=0.9238702912817508, total=   0.0s
[CV] colsample_bytree=0.9420989706632965, gamma=4.84743784631088, learning_rate=0.2971593794925696, max_depth=12, min_child_weight=61.21758661968412, n_estimators=21, reg_alpha=3.5194407158563146, subsample=0.9238702912817508 
[CV]  colsample_bytree=0.9420989706632965, gamma=4.84743784631088, learning_rate=0.2971593794925696, max_depth=12, min_child_weight=61.21758661968412, n_estimators=21, reg_alpha=3.5194407158563146, subsample=0.9238702912817508, total=   0.0s
[CV] colsample_bytree=0.9420989706632965, gamma=4.84743784631088, learning_rate=0.2971593794925696, max_depth=12, min_child_weight=61.21758661968412, n_estimators=21, reg_alpha=3.5194407158563146, subsample=0.9238702912817508 
[CV]  colsample_bytree=0.9420989706632965, gamma=4.84743784631088, learning_rate=0.2971593794925696, max_depth=12, min_child_weight=61.21758661968412, n_estimators=21, reg_alpha=3.5194407158563146, subsample=0.9238702912817508, total=   0.0s
[CV] colsample_bytree=0.9526617655226074, gamma=3.2881926543503193, learning_rate=0.18762607630713307, max_depth=4, min_child_weight=51.62657285647335, n_estimators=10, reg_alpha=116.77148513799638, subsample=0.8462795211256848 
[CV]  colsample_bytree=0.9526617655226074, gamma=3.2881926543503193, learning_rate=0.18762607630713307, max_depth=4, min_child_weight=51.62657285647335, n_estimators=10, reg_alpha=116.77148513799638, subsample=0.8462795211256848, total=   0.0s
[CV] colsample_bytree=0.9526617655226074, gamma=3.2881926543503193, learning_rate=0.18762607630713307, max_depth=4, min_child_weight=51.62657285647335, n_estimators=10, reg_alpha=116.77148513799638, subsample=0.8462795211256848 
[CV]  colsample_bytree=0.9526617655226074, gamma=3.2881926543503193, learning_rate=0.18762607630713307, max_depth=4, min_child_weight=51.62657285647335, n_estimators=10, reg_alpha=116.77148513799638, subsample=0.8462795211256848, total=   0.0s
[CV] colsample_bytree=0.9526617655226074, gamma=3.2881926543503193, learning_rate=0.18762607630713307, max_depth=4, min_child_weight=51.62657285647335, n_estimators=10, reg_alpha=116.77148513799638, subsample=0.8462795211256848 
[CV]  colsample_bytree=0.9526617655226074, gamma=3.2881926543503193, learning_rate=0.18762607630713307, max_depth=4, min_child_weight=51.62657285647335, n_estimators=10, reg_alpha=116.77148513799638, subsample=0.8462795211256848, total=   0.0s
[CV] colsample_bytree=0.9952208110058465, gamma=3.6013266403457846, learning_rate=0.08040703363883908, max_depth=5, min_child_weight=20.626252034146358, n_estimators=8, reg_alpha=90.17286185247367, subsample=0.9516457485929473 
[CV]  colsample_bytree=0.9952208110058465, gamma=3.6013266403457846, learning_rate=0.08040703363883908, max_depth=5, min_child_weight=20.626252034146358, n_estimators=8, reg_alpha=90.17286185247367, subsample=0.9516457485929473, total=   0.0s
[CV] colsample_bytree=0.9952208110058465, gamma=3.6013266403457846, learning_rate=0.08040703363883908, max_depth=5, min_child_weight=20.626252034146358, n_estimators=8, reg_alpha=90.17286185247367, subsample=0.9516457485929473 
[CV]  colsample_bytree=0.9952208110058465, gamma=3.6013266403457846, learning_rate=0.08040703363883908, max_depth=5, min_child_weight=20.626252034146358, n_estimators=8, reg_alpha=90.17286185247367, subsample=0.9516457485929473, total=   0.0s
[CV] colsample_bytree=0.9952208110058465, gamma=3.6013266403457846, learning_rate=0.08040703363883908, max_depth=5, min_child_weight=20.626252034146358, n_estimators=8, reg_alpha=90.17286185247367, subsample=0.9516457485929473 
[CV]  colsample_bytree=0.9952208110058465, gamma=3.6013266403457846, learning_rate=0.08040703363883908, max_depth=5, min_child_weight=20.626252034146358, n_estimators=8, reg_alpha=90.17286185247367, subsample=0.9516457485929473, total=   0.0s
[CV] colsample_bytree=0.9468350325622289, gamma=5.306974384761395, learning_rate=0.24764650635853003, max_depth=8, min_child_weight=11.248928334409172, n_estimators=34, reg_alpha=93.97842249401404, subsample=0.9057809658772924 
[CV]  colsample_bytree=0.9468350325622289, gamma=5.306974384761395, learning_rate=0.24764650635853003, max_depth=8, min_child_weight=11.248928334409172, n_estimators=34, reg_alpha=93.97842249401404, subsample=0.9057809658772924, total=   0.0s
[CV] colsample_bytree=0.9468350325622289, gamma=5.306974384761395, learning_rate=0.24764650635853003, max_depth=8, min_child_weight=11.248928334409172, n_estimators=34, reg_alpha=93.97842249401404, subsample=0.9057809658772924 
[CV]  colsample_bytree=0.9468350325622289, gamma=5.306974384761395, learning_rate=0.24764650635853003, max_depth=8, min_child_weight=11.248928334409172, n_estimators=34, reg_alpha=93.97842249401404, subsample=0.9057809658772924, total=   0.0s
[CV] colsample_bytree=0.9468350325622289, gamma=5.306974384761395, learning_rate=0.24764650635853003, max_depth=8, min_child_weight=11.248928334409172, n_estimators=34, reg_alpha=93.97842249401404, subsample=0.9057809658772924 
[CV]  colsample_bytree=0.9468350325622289, gamma=5.306974384761395, learning_rate=0.24764650635853003, max_depth=8, min_child_weight=11.248928334409172, n_estimators=34, reg_alpha=93.97842249401404, subsample=0.9057809658772924, total=   0.0s
[CV] colsample_bytree=0.7855223860034094, gamma=6.865631792310277, learning_rate=0.41797889109586567, max_depth=22, min_child_weight=10.511733271788604, n_estimators=8, reg_alpha=43.86550084469945, subsample=0.97371604131475 
[CV]  colsample_bytree=0.7855223860034094, gamma=6.865631792310277, learning_rate=0.41797889109586567, max_depth=22, min_child_weight=10.511733271788604, n_estimators=8, reg_alpha=43.86550084469945, subsample=0.97371604131475, total=   0.0s
[CV] colsample_bytree=0.7855223860034094, gamma=6.865631792310277, learning_rate=0.41797889109586567, max_depth=22, min_child_weight=10.511733271788604, n_estimators=8, reg_alpha=43.86550084469945, subsample=0.97371604131475 
[CV]  colsample_bytree=0.7855223860034094, gamma=6.865631792310277, learning_rate=0.41797889109586567, max_depth=22, min_child_weight=10.511733271788604, n_estimators=8, reg_alpha=43.86550084469945, subsample=0.97371604131475, total=   0.0s
[CV] colsample_bytree=0.7855223860034094, gamma=6.865631792310277, learning_rate=0.41797889109586567, max_depth=22, min_child_weight=10.511733271788604, n_estimators=8, reg_alpha=43.86550084469945, subsample=0.97371604131475 
[CV]  colsample_bytree=0.7855223860034094, gamma=6.865631792310277, learning_rate=0.41797889109586567, max_depth=22, min_child_weight=10.511733271788604, n_estimators=8, reg_alpha=43.86550084469945, subsample=0.97371604131475, total=   0.0s
[CV] colsample_bytree=0.9093127523974813, gamma=2.6126421140109413, learning_rate=0.44282223290137973, max_depth=38, min_child_weight=20.448993143029096, n_estimators=8, reg_alpha=71.09544088465093, subsample=0.5126573340214049 
[CV]  colsample_bytree=0.9093127523974813, gamma=2.6126421140109413, learning_rate=0.44282223290137973, max_depth=38, min_child_weight=20.448993143029096, n_estimators=8, reg_alpha=71.09544088465093, subsample=0.5126573340214049, total=   0.0s
[CV] colsample_bytree=0.9093127523974813, gamma=2.6126421140109413, learning_rate=0.44282223290137973, max_depth=38, min_child_weight=20.448993143029096, n_estimators=8, reg_alpha=71.09544088465093, subsample=0.5126573340214049 
[CV]  colsample_bytree=0.9093127523974813, gamma=2.6126421140109413, learning_rate=0.44282223290137973, max_depth=38, min_child_weight=20.448993143029096, n_estimators=8, reg_alpha=71.09544088465093, subsample=0.5126573340214049, total=   0.0s
[CV] colsample_bytree=0.9093127523974813, gamma=2.6126421140109413, learning_rate=0.44282223290137973, max_depth=38, min_child_weight=20.448993143029096, n_estimators=8, reg_alpha=71.09544088465093, subsample=0.5126573340214049 
[CV]  colsample_bytree=0.9093127523974813, gamma=2.6126421140109413, learning_rate=0.44282223290137973, max_depth=38, min_child_weight=20.448993143029096, n_estimators=8, reg_alpha=71.09544088465093, subsample=0.5126573340214049, total=   0.0s
[CV] colsample_bytree=0.869127063656987, gamma=3.748759435368594, learning_rate=0.35812901013198306, max_depth=28, min_child_weight=60.405548175429566, n_estimators=27, reg_alpha=3.1917422946152985, subsample=0.9734154918059101 
[CV]  colsample_bytree=0.869127063656987, gamma=3.748759435368594, learning_rate=0.35812901013198306, max_depth=28, min_child_weight=60.405548175429566, n_estimators=27, reg_alpha=3.1917422946152985, subsample=0.9734154918059101, total=   0.0s
[CV] colsample_bytree=0.869127063656987, gamma=3.748759435368594, learning_rate=0.35812901013198306, max_depth=28, min_child_weight=60.405548175429566, n_estimators=27, reg_alpha=3.1917422946152985, subsample=0.9734154918059101 
[CV]  colsample_bytree=0.869127063656987, gamma=3.748759435368594, learning_rate=0.35812901013198306, max_depth=28, min_child_weight=60.405548175429566, n_estimators=27, reg_alpha=3.1917422946152985, subsample=0.9734154918059101, total=   0.0s
[CV] colsample_bytree=0.869127063656987, gamma=3.748759435368594, learning_rate=0.35812901013198306, max_depth=28, min_child_weight=60.405548175429566, n_estimators=27, reg_alpha=3.1917422946152985, subsample=0.9734154918059101 
[CV]  colsample_bytree=0.869127063656987, gamma=3.748759435368594, learning_rate=0.35812901013198306, max_depth=28, min_child_weight=60.405548175429566, n_estimators=27, reg_alpha=3.1917422946152985, subsample=0.9734154918059101, total=   0.0s
[CV] colsample_bytree=0.959149827359648, gamma=9.099835148679354, learning_rate=0.12146655184974203, max_depth=33, min_child_weight=16.8934760909329, n_estimators=15, reg_alpha=6.514209188142021, subsample=0.844802145394611 
[CV]  colsample_bytree=0.959149827359648, gamma=9.099835148679354, learning_rate=0.12146655184974203, max_depth=33, min_child_weight=16.8934760909329, n_estimators=15, reg_alpha=6.514209188142021, subsample=0.844802145394611, total=   0.0s
[CV] colsample_bytree=0.959149827359648, gamma=9.099835148679354, learning_rate=0.12146655184974203, max_depth=33, min_child_weight=16.8934760909329, n_estimators=15, reg_alpha=6.514209188142021, subsample=0.844802145394611 
[CV]  colsample_bytree=0.959149827359648, gamma=9.099835148679354, learning_rate=0.12146655184974203, max_depth=33, min_child_weight=16.8934760909329, n_estimators=15, reg_alpha=6.514209188142021, subsample=0.844802145394611, total=   0.0s
[CV] colsample_bytree=0.959149827359648, gamma=9.099835148679354, learning_rate=0.12146655184974203, max_depth=33, min_child_weight=16.8934760909329, n_estimators=15, reg_alpha=6.514209188142021, subsample=0.844802145394611 
[CV]  colsample_bytree=0.959149827359648, gamma=9.099835148679354, learning_rate=0.12146655184974203, max_depth=33, min_child_weight=16.8934760909329, n_estimators=15, reg_alpha=6.514209188142021, subsample=0.844802145394611, total=   0.0s
[CV] colsample_bytree=0.9136250407720526, gamma=5.9858045984580235, learning_rate=0.1901816865055907, max_depth=24, min_child_weight=12.518161908868006, n_estimators=17, reg_alpha=117.64662065867708, subsample=0.8673342615184937 
[CV]  colsample_bytree=0.9136250407720526, gamma=5.9858045984580235, learning_rate=0.1901816865055907, max_depth=24, min_child_weight=12.518161908868006, n_estimators=17, reg_alpha=117.64662065867708, subsample=0.8673342615184937, total=   0.0s
[CV] colsample_bytree=0.9136250407720526, gamma=5.9858045984580235, learning_rate=0.1901816865055907, max_depth=24, min_child_weight=12.518161908868006, n_estimators=17, reg_alpha=117.64662065867708, subsample=0.8673342615184937 
[CV]  colsample_bytree=0.9136250407720526, gamma=5.9858045984580235, learning_rate=0.1901816865055907, max_depth=24, min_child_weight=12.518161908868006, n_estimators=17, reg_alpha=117.64662065867708, subsample=0.8673342615184937, total=   0.0s
[CV] colsample_bytree=0.9136250407720526, gamma=5.9858045984580235, learning_rate=0.1901816865055907, max_depth=24, min_child_weight=12.518161908868006, n_estimators=17, reg_alpha=117.64662065867708, subsample=0.8673342615184937 
[CV]  colsample_bytree=0.9136250407720526, gamma=5.9858045984580235, learning_rate=0.1901816865055907, max_depth=24, min_child_weight=12.518161908868006, n_estimators=17, reg_alpha=117.64662065867708, subsample=0.8673342615184937, total=   0.0s
[CV] colsample_bytree=0.9776013124075571, gamma=9.500317136335445, learning_rate=0.10019843409353944, max_depth=37, min_child_weight=120.01650549069952, n_estimators=31, reg_alpha=72.78749281655675, subsample=0.9616072201019219 
[CV]  colsample_bytree=0.9776013124075571, gamma=9.500317136335445, learning_rate=0.10019843409353944, max_depth=37, min_child_weight=120.01650549069952, n_estimators=31, reg_alpha=72.78749281655675, subsample=0.9616072201019219, total=   0.0s
[CV] colsample_bytree=0.9776013124075571, gamma=9.500317136335445, learning_rate=0.10019843409353944, max_depth=37, min_child_weight=120.01650549069952, n_estimators=31, reg_alpha=72.78749281655675, subsample=0.9616072201019219 
[CV]  colsample_bytree=0.9776013124075571, gamma=9.500317136335445, learning_rate=0.10019843409353944, max_depth=37, min_child_weight=120.01650549069952, n_estimators=31, reg_alpha=72.78749281655675, subsample=0.9616072201019219, total=   0.0s
[CV] colsample_bytree=0.9776013124075571, gamma=9.500317136335445, learning_rate=0.10019843409353944, max_depth=37, min_child_weight=120.01650549069952, n_estimators=31, reg_alpha=72.78749281655675, subsample=0.9616072201019219 
[CV]  colsample_bytree=0.9776013124075571, gamma=9.500317136335445, learning_rate=0.10019843409353944, max_depth=37, min_child_weight=120.01650549069952, n_estimators=31, reg_alpha=72.78749281655675, subsample=0.9616072201019219, total=   0.0s
[CV] colsample_bytree=0.9820437330363218, gamma=8.479393924434762, learning_rate=0.15065565694234112, max_depth=32, min_child_weight=41.866048375706455, n_estimators=35, reg_alpha=148.63623768478072, subsample=0.8460776026451077 
[CV]  colsample_bytree=0.9820437330363218, gamma=8.479393924434762, learning_rate=0.15065565694234112, max_depth=32, min_child_weight=41.866048375706455, n_estimators=35, reg_alpha=148.63623768478072, subsample=0.8460776026451077, total=   0.0s
[CV] colsample_bytree=0.9820437330363218, gamma=8.479393924434762, learning_rate=0.15065565694234112, max_depth=32, min_child_weight=41.866048375706455, n_estimators=35, reg_alpha=148.63623768478072, subsample=0.8460776026451077 
[CV]  colsample_bytree=0.9820437330363218, gamma=8.479393924434762, learning_rate=0.15065565694234112, max_depth=32, min_child_weight=41.866048375706455, n_estimators=35, reg_alpha=148.63623768478072, subsample=0.8460776026451077, total=   0.0s
[CV] colsample_bytree=0.9820437330363218, gamma=8.479393924434762, learning_rate=0.15065565694234112, max_depth=32, min_child_weight=41.866048375706455, n_estimators=35, reg_alpha=148.63623768478072, subsample=0.8460776026451077 
[CV]  colsample_bytree=0.9820437330363218, gamma=8.479393924434762, learning_rate=0.15065565694234112, max_depth=32, min_child_weight=41.866048375706455, n_estimators=35, reg_alpha=148.63623768478072, subsample=0.8460776026451077, total=   0.0s
[CV] colsample_bytree=0.9990000765046462, gamma=8.227300054219356, learning_rate=0.3944846553467178, max_depth=31, min_child_weight=44.27167861486731, n_estimators=7, reg_alpha=38.00909720209803, subsample=0.8555345007117244 
[CV]  colsample_bytree=0.9990000765046462, gamma=8.227300054219356, learning_rate=0.3944846553467178, max_depth=31, min_child_weight=44.27167861486731, n_estimators=7, reg_alpha=38.00909720209803, subsample=0.8555345007117244, total=   0.0s
[CV] colsample_bytree=0.9990000765046462, gamma=8.227300054219356, learning_rate=0.3944846553467178, max_depth=31, min_child_weight=44.27167861486731, n_estimators=7, reg_alpha=38.00909720209803, subsample=0.8555345007117244 
[CV]  colsample_bytree=0.9990000765046462, gamma=8.227300054219356, learning_rate=0.3944846553467178, max_depth=31, min_child_weight=44.27167861486731, n_estimators=7, reg_alpha=38.00909720209803, subsample=0.8555345007117244, total=   0.0s
[CV] colsample_bytree=0.9990000765046462, gamma=8.227300054219356, learning_rate=0.3944846553467178, max_depth=31, min_child_weight=44.27167861486731, n_estimators=7, reg_alpha=38.00909720209803, subsample=0.8555345007117244 
[CV]  colsample_bytree=0.9990000765046462, gamma=8.227300054219356, learning_rate=0.3944846553467178, max_depth=31, min_child_weight=44.27167861486731, n_estimators=7, reg_alpha=38.00909720209803, subsample=0.8555345007117244, total=   0.0s
[CV] colsample_bytree=0.8738815719613293, gamma=0.6806776224069766, learning_rate=0.204639451222843, max_depth=38, min_child_weight=10.508386556149583, n_estimators=21, reg_alpha=140.11635207680172, subsample=0.9001976575226052 
[CV]  colsample_bytree=0.8738815719613293, gamma=0.6806776224069766, learning_rate=0.204639451222843, max_depth=38, min_child_weight=10.508386556149583, n_estimators=21, reg_alpha=140.11635207680172, subsample=0.9001976575226052, total=   0.0s
[CV] colsample_bytree=0.8738815719613293, gamma=0.6806776224069766, learning_rate=0.204639451222843, max_depth=38, min_child_weight=10.508386556149583, n_estimators=21, reg_alpha=140.11635207680172, subsample=0.9001976575226052 
[CV]  colsample_bytree=0.8738815719613293, gamma=0.6806776224069766, learning_rate=0.204639451222843, max_depth=38, min_child_weight=10.508386556149583, n_estimators=21, reg_alpha=140.11635207680172, subsample=0.9001976575226052, total=   0.0s
[CV] colsample_bytree=0.8738815719613293, gamma=0.6806776224069766, learning_rate=0.204639451222843, max_depth=38, min_child_weight=10.508386556149583, n_estimators=21, reg_alpha=140.11635207680172, subsample=0.9001976575226052 
[CV]  colsample_bytree=0.8738815719613293, gamma=0.6806776224069766, learning_rate=0.204639451222843, max_depth=38, min_child_weight=10.508386556149583, n_estimators=21, reg_alpha=140.11635207680172, subsample=0.9001976575226052, total=   0.0s
[CV] colsample_bytree=0.9997682499504452, gamma=0.14003556274473028, learning_rate=0.13704444107956004, max_depth=3, min_child_weight=59.497613362937585, n_estimators=23, reg_alpha=26.084460047321635, subsample=0.9745800960774649 
[CV]  colsample_bytree=0.9997682499504452, gamma=0.14003556274473028, learning_rate=0.13704444107956004, max_depth=3, min_child_weight=59.497613362937585, n_estimators=23, reg_alpha=26.084460047321635, subsample=0.9745800960774649, total=   0.0s
[CV] colsample_bytree=0.9997682499504452, gamma=0.14003556274473028, learning_rate=0.13704444107956004, max_depth=3, min_child_weight=59.497613362937585, n_estimators=23, reg_alpha=26.084460047321635, subsample=0.9745800960774649 
[CV]  colsample_bytree=0.9997682499504452, gamma=0.14003556274473028, learning_rate=0.13704444107956004, max_depth=3, min_child_weight=59.497613362937585, n_estimators=23, reg_alpha=26.084460047321635, subsample=0.9745800960774649, total=   0.0s
[CV] colsample_bytree=0.9997682499504452, gamma=0.14003556274473028, learning_rate=0.13704444107956004, max_depth=3, min_child_weight=59.497613362937585, n_estimators=23, reg_alpha=26.084460047321635, subsample=0.9745800960774649 
[CV]  colsample_bytree=0.9997682499504452, gamma=0.14003556274473028, learning_rate=0.13704444107956004, max_depth=3, min_child_weight=59.497613362937585, n_estimators=23, reg_alpha=26.084460047321635, subsample=0.9745800960774649, total=   0.0s
[CV] colsample_bytree=0.9620185833498037, gamma=4.0692199412935794, learning_rate=0.44008893697492485, max_depth=26, min_child_weight=7.910222863563258, n_estimators=26, reg_alpha=37.20856994634564, subsample=0.9640008487287339 
[CV]  colsample_bytree=0.9620185833498037, gamma=4.0692199412935794, learning_rate=0.44008893697492485, max_depth=26, min_child_weight=7.910222863563258, n_estimators=26, reg_alpha=37.20856994634564, subsample=0.9640008487287339, total=   0.0s
[CV] colsample_bytree=0.9620185833498037, gamma=4.0692199412935794, learning_rate=0.44008893697492485, max_depth=26, min_child_weight=7.910222863563258, n_estimators=26, reg_alpha=37.20856994634564, subsample=0.9640008487287339 
[CV]  colsample_bytree=0.9620185833498037, gamma=4.0692199412935794, learning_rate=0.44008893697492485, max_depth=26, min_child_weight=7.910222863563258, n_estimators=26, reg_alpha=37.20856994634564, subsample=0.9640008487287339, total=   0.0s
[CV] colsample_bytree=0.9620185833498037, gamma=4.0692199412935794, learning_rate=0.44008893697492485, max_depth=26, min_child_weight=7.910222863563258, n_estimators=26, reg_alpha=37.20856994634564, subsample=0.9640008487287339 
[CV]  colsample_bytree=0.9620185833498037, gamma=4.0692199412935794, learning_rate=0.44008893697492485, max_depth=26, min_child_weight=7.910222863563258, n_estimators=26, reg_alpha=37.20856994634564, subsample=0.9640008487287339, total=   0.0s
[CV] colsample_bytree=0.9985710074383568, gamma=6.137196744254004, learning_rate=0.06247863679340147, max_depth=21, min_child_weight=15.643165353514888, n_estimators=24, reg_alpha=44.81080275678912, subsample=0.78115836934102 
[CV]  colsample_bytree=0.9985710074383568, gamma=6.137196744254004, learning_rate=0.06247863679340147, max_depth=21, min_child_weight=15.643165353514888, n_estimators=24, reg_alpha=44.81080275678912, subsample=0.78115836934102, total=   0.0s
[CV] colsample_bytree=0.9985710074383568, gamma=6.137196744254004, learning_rate=0.06247863679340147, max_depth=21, min_child_weight=15.643165353514888, n_estimators=24, reg_alpha=44.81080275678912, subsample=0.78115836934102 
[CV]  colsample_bytree=0.9985710074383568, gamma=6.137196744254004, learning_rate=0.06247863679340147, max_depth=21, min_child_weight=15.643165353514888, n_estimators=24, reg_alpha=44.81080275678912, subsample=0.78115836934102, total=   0.0s
[CV] colsample_bytree=0.9985710074383568, gamma=6.137196744254004, learning_rate=0.06247863679340147, max_depth=21, min_child_weight=15.643165353514888, n_estimators=24, reg_alpha=44.81080275678912, subsample=0.78115836934102 
[CV]  colsample_bytree=0.9985710074383568, gamma=6.137196744254004, learning_rate=0.06247863679340147, max_depth=21, min_child_weight=15.643165353514888, n_estimators=24, reg_alpha=44.81080275678912, subsample=0.78115836934102, total=   0.0s
[CV] colsample_bytree=0.7692563952836217, gamma=2.4699335469458283, learning_rate=0.27182224227820995, max_depth=7, min_child_weight=127.46797423947851, n_estimators=16, reg_alpha=46.93080163156149, subsample=0.9690271672470472 
[CV]  colsample_bytree=0.7692563952836217, gamma=2.4699335469458283, learning_rate=0.27182224227820995, max_depth=7, min_child_weight=127.46797423947851, n_estimators=16, reg_alpha=46.93080163156149, subsample=0.9690271672470472, total=   0.0s
[CV] colsample_bytree=0.7692563952836217, gamma=2.4699335469458283, learning_rate=0.27182224227820995, max_depth=7, min_child_weight=127.46797423947851, n_estimators=16, reg_alpha=46.93080163156149, subsample=0.9690271672470472 
[CV]  colsample_bytree=0.7692563952836217, gamma=2.4699335469458283, learning_rate=0.27182224227820995, max_depth=7, min_child_weight=127.46797423947851, n_estimators=16, reg_alpha=46.93080163156149, subsample=0.9690271672470472, total=   0.0s
[CV] colsample_bytree=0.7692563952836217, gamma=2.4699335469458283, learning_rate=0.27182224227820995, max_depth=7, min_child_weight=127.46797423947851, n_estimators=16, reg_alpha=46.93080163156149, subsample=0.9690271672470472 
[CV]  colsample_bytree=0.7692563952836217, gamma=2.4699335469458283, learning_rate=0.27182224227820995, max_depth=7, min_child_weight=127.46797423947851, n_estimators=16, reg_alpha=46.93080163156149, subsample=0.9690271672470472, total=   0.0s
[CV] colsample_bytree=0.9578430759392847, gamma=7.797442954826, learning_rate=0.11989452513096298, max_depth=23, min_child_weight=73.84840457229788, n_estimators=38, reg_alpha=63.07494325399403, subsample=0.8849652689489391 
[CV]  colsample_bytree=0.9578430759392847, gamma=7.797442954826, learning_rate=0.11989452513096298, max_depth=23, min_child_weight=73.84840457229788, n_estimators=38, reg_alpha=63.07494325399403, subsample=0.8849652689489391, total=   0.0s
[CV] colsample_bytree=0.9578430759392847, gamma=7.797442954826, learning_rate=0.11989452513096298, max_depth=23, min_child_weight=73.84840457229788, n_estimators=38, reg_alpha=63.07494325399403, subsample=0.8849652689489391 
[CV]  colsample_bytree=0.9578430759392847, gamma=7.797442954826, learning_rate=0.11989452513096298, max_depth=23, min_child_weight=73.84840457229788, n_estimators=38, reg_alpha=63.07494325399403, subsample=0.8849652689489391, total=   0.0s
[CV] colsample_bytree=0.9578430759392847, gamma=7.797442954826, learning_rate=0.11989452513096298, max_depth=23, min_child_weight=73.84840457229788, n_estimators=38, reg_alpha=63.07494325399403, subsample=0.8849652689489391 
[CV]  colsample_bytree=0.9578430759392847, gamma=7.797442954826, learning_rate=0.11989452513096298, max_depth=23, min_child_weight=73.84840457229788, n_estimators=38, reg_alpha=63.07494325399403, subsample=0.8849652689489391, total=   0.0s
[CV] colsample_bytree=0.9577483201693804, gamma=3.719221216676872, learning_rate=0.1921231646480489, max_depth=36, min_child_weight=49.93562742397889, n_estimators=38, reg_alpha=24.601536714807573, subsample=0.9736157364282024 
[CV]  colsample_bytree=0.9577483201693804, gamma=3.719221216676872, learning_rate=0.1921231646480489, max_depth=36, min_child_weight=49.93562742397889, n_estimators=38, reg_alpha=24.601536714807573, subsample=0.9736157364282024, total=   0.1s
[CV] colsample_bytree=0.9577483201693804, gamma=3.719221216676872, learning_rate=0.1921231646480489, max_depth=36, min_child_weight=49.93562742397889, n_estimators=38, reg_alpha=24.601536714807573, subsample=0.9736157364282024 
[CV]  colsample_bytree=0.9577483201693804, gamma=3.719221216676872, learning_rate=0.1921231646480489, max_depth=36, min_child_weight=49.93562742397889, n_estimators=38, reg_alpha=24.601536714807573, subsample=0.9736157364282024, total=   0.1s
[CV] colsample_bytree=0.9577483201693804, gamma=3.719221216676872, learning_rate=0.1921231646480489, max_depth=36, min_child_weight=49.93562742397889, n_estimators=38, reg_alpha=24.601536714807573, subsample=0.9736157364282024 
[CV]  colsample_bytree=0.9577483201693804, gamma=3.719221216676872, learning_rate=0.1921231646480489, max_depth=36, min_child_weight=49.93562742397889, n_estimators=38, reg_alpha=24.601536714807573, subsample=0.9736157364282024, total=   0.0s
[CV] colsample_bytree=0.9062642108186547, gamma=9.82248957923346, learning_rate=0.4270179791120658, max_depth=24, min_child_weight=59.059560964959545, n_estimators=22, reg_alpha=20.961011968115272, subsample=0.8487860764282807 
[CV]  colsample_bytree=0.9062642108186547, gamma=9.82248957923346, learning_rate=0.4270179791120658, max_depth=24, min_child_weight=59.059560964959545, n_estimators=22, reg_alpha=20.961011968115272, subsample=0.8487860764282807, total=   0.0s
[CV] colsample_bytree=0.9062642108186547, gamma=9.82248957923346, learning_rate=0.4270179791120658, max_depth=24, min_child_weight=59.059560964959545, n_estimators=22, reg_alpha=20.961011968115272, subsample=0.8487860764282807 
[CV]  colsample_bytree=0.9062642108186547, gamma=9.82248957923346, learning_rate=0.4270179791120658, max_depth=24, min_child_weight=59.059560964959545, n_estimators=22, reg_alpha=20.961011968115272, subsample=0.8487860764282807, total=   0.0s
[CV] colsample_bytree=0.9062642108186547, gamma=9.82248957923346, learning_rate=0.4270179791120658, max_depth=24, min_child_weight=59.059560964959545, n_estimators=22, reg_alpha=20.961011968115272, subsample=0.8487860764282807 
[CV]  colsample_bytree=0.9062642108186547, gamma=9.82248957923346, learning_rate=0.4270179791120658, max_depth=24, min_child_weight=59.059560964959545, n_estimators=22, reg_alpha=20.961011968115272, subsample=0.8487860764282807, total=   0.0s
[CV] colsample_bytree=0.9822102559934811, gamma=5.096132942175586, learning_rate=0.16877883431476182, max_depth=13, min_child_weight=84.02904188235755, n_estimators=34, reg_alpha=6.662360225871086, subsample=0.7707132413386909 
[CV]  colsample_bytree=0.9822102559934811, gamma=5.096132942175586, learning_rate=0.16877883431476182, max_depth=13, min_child_weight=84.02904188235755, n_estimators=34, reg_alpha=6.662360225871086, subsample=0.7707132413386909, total=   0.0s
[CV] colsample_bytree=0.9822102559934811, gamma=5.096132942175586, learning_rate=0.16877883431476182, max_depth=13, min_child_weight=84.02904188235755, n_estimators=34, reg_alpha=6.662360225871086, subsample=0.7707132413386909 
[CV]  colsample_bytree=0.9822102559934811, gamma=5.096132942175586, learning_rate=0.16877883431476182, max_depth=13, min_child_weight=84.02904188235755, n_estimators=34, reg_alpha=6.662360225871086, subsample=0.7707132413386909, total=   0.0s
[CV] colsample_bytree=0.9822102559934811, gamma=5.096132942175586, learning_rate=0.16877883431476182, max_depth=13, min_child_weight=84.02904188235755, n_estimators=34, reg_alpha=6.662360225871086, subsample=0.7707132413386909 
[CV]  colsample_bytree=0.9822102559934811, gamma=5.096132942175586, learning_rate=0.16877883431476182, max_depth=13, min_child_weight=84.02904188235755, n_estimators=34, reg_alpha=6.662360225871086, subsample=0.7707132413386909, total=   0.0s
[CV] colsample_bytree=0.9086700861484762, gamma=5.656621125439275, learning_rate=0.23746633506450082, max_depth=27, min_child_weight=52.619928019401854, n_estimators=29, reg_alpha=69.53943033083029, subsample=0.8121353818851516 
[CV]  colsample_bytree=0.9086700861484762, gamma=5.656621125439275, learning_rate=0.23746633506450082, max_depth=27, min_child_weight=52.619928019401854, n_estimators=29, reg_alpha=69.53943033083029, subsample=0.8121353818851516, total=   0.0s
[CV] colsample_bytree=0.9086700861484762, gamma=5.656621125439275, learning_rate=0.23746633506450082, max_depth=27, min_child_weight=52.619928019401854, n_estimators=29, reg_alpha=69.53943033083029, subsample=0.8121353818851516 
[CV]  colsample_bytree=0.9086700861484762, gamma=5.656621125439275, learning_rate=0.23746633506450082, max_depth=27, min_child_weight=52.619928019401854, n_estimators=29, reg_alpha=69.53943033083029, subsample=0.8121353818851516, total=   0.0s
[CV] colsample_bytree=0.9086700861484762, gamma=5.656621125439275, learning_rate=0.23746633506450082, max_depth=27, min_child_weight=52.619928019401854, n_estimators=29, reg_alpha=69.53943033083029, subsample=0.8121353818851516 
[CV]  colsample_bytree=0.9086700861484762, gamma=5.656621125439275, learning_rate=0.23746633506450082, max_depth=27, min_child_weight=52.619928019401854, n_estimators=29, reg_alpha=69.53943033083029, subsample=0.8121353818851516, total=   0.0s
[CV] colsample_bytree=0.9930835787468935, gamma=2.8462388100694214, learning_rate=0.07441467244786844, max_depth=33, min_child_weight=12.744852760095585, n_estimators=3, reg_alpha=23.787452591152984, subsample=0.8689177375584267 
[CV]  colsample_bytree=0.9930835787468935, gamma=2.8462388100694214, learning_rate=0.07441467244786844, max_depth=33, min_child_weight=12.744852760095585, n_estimators=3, reg_alpha=23.787452591152984, subsample=0.8689177375584267, total=   0.0s
[CV] colsample_bytree=0.9930835787468935, gamma=2.8462388100694214, learning_rate=0.07441467244786844, max_depth=33, min_child_weight=12.744852760095585, n_estimators=3, reg_alpha=23.787452591152984, subsample=0.8689177375584267 
[CV]  colsample_bytree=0.9930835787468935, gamma=2.8462388100694214, learning_rate=0.07441467244786844, max_depth=33, min_child_weight=12.744852760095585, n_estimators=3, reg_alpha=23.787452591152984, subsample=0.8689177375584267, total=   0.0s
[CV] colsample_bytree=0.9930835787468935, gamma=2.8462388100694214, learning_rate=0.07441467244786844, max_depth=33, min_child_weight=12.744852760095585, n_estimators=3, reg_alpha=23.787452591152984, subsample=0.8689177375584267 
[CV]  colsample_bytree=0.9930835787468935, gamma=2.8462388100694214, learning_rate=0.07441467244786844, max_depth=33, min_child_weight=12.744852760095585, n_estimators=3, reg_alpha=23.787452591152984, subsample=0.8689177375584267, total=   0.0s
[CV] colsample_bytree=0.9478168458985571, gamma=0.28490849491567216, learning_rate=0.3076928079516127, max_depth=18, min_child_weight=13.016643231882238, n_estimators=34, reg_alpha=178.96307930826396, subsample=0.8963554809961419 
[CV]  colsample_bytree=0.9478168458985571, gamma=0.28490849491567216, learning_rate=0.3076928079516127, max_depth=18, min_child_weight=13.016643231882238, n_estimators=34, reg_alpha=178.96307930826396, subsample=0.8963554809961419, total=   0.0s
[CV] colsample_bytree=0.9478168458985571, gamma=0.28490849491567216, learning_rate=0.3076928079516127, max_depth=18, min_child_weight=13.016643231882238, n_estimators=34, reg_alpha=178.96307930826396, subsample=0.8963554809961419 
[CV]  colsample_bytree=0.9478168458985571, gamma=0.28490849491567216, learning_rate=0.3076928079516127, max_depth=18, min_child_weight=13.016643231882238, n_estimators=34, reg_alpha=178.96307930826396, subsample=0.8963554809961419, total=   0.0s
[CV] colsample_bytree=0.9478168458985571, gamma=0.28490849491567216, learning_rate=0.3076928079516127, max_depth=18, min_child_weight=13.016643231882238, n_estimators=34, reg_alpha=178.96307930826396, subsample=0.8963554809961419 
[CV]  colsample_bytree=0.9478168458985571, gamma=0.28490849491567216, learning_rate=0.3076928079516127, max_depth=18, min_child_weight=13.016643231882238, n_estimators=34, reg_alpha=178.96307930826396, subsample=0.8963554809961419, total=   0.0s
[CV] colsample_bytree=0.857809552604072, gamma=6.741718974333468, learning_rate=0.1413670964405497, max_depth=8, min_child_weight=35.77432109356246, n_estimators=16, reg_alpha=40.64238067539015, subsample=0.6009037853772013 
[CV]  colsample_bytree=0.857809552604072, gamma=6.741718974333468, learning_rate=0.1413670964405497, max_depth=8, min_child_weight=35.77432109356246, n_estimators=16, reg_alpha=40.64238067539015, subsample=0.6009037853772013, total=   0.0s
[CV] colsample_bytree=0.857809552604072, gamma=6.741718974333468, learning_rate=0.1413670964405497, max_depth=8, min_child_weight=35.77432109356246, n_estimators=16, reg_alpha=40.64238067539015, subsample=0.6009037853772013 
[CV]  colsample_bytree=0.857809552604072, gamma=6.741718974333468, learning_rate=0.1413670964405497, max_depth=8, min_child_weight=35.77432109356246, n_estimators=16, reg_alpha=40.64238067539015, subsample=0.6009037853772013, total=   0.0s
[CV] colsample_bytree=0.857809552604072, gamma=6.741718974333468, learning_rate=0.1413670964405497, max_depth=8, min_child_weight=35.77432109356246, n_estimators=16, reg_alpha=40.64238067539015, subsample=0.6009037853772013 
[CV]  colsample_bytree=0.857809552604072, gamma=6.741718974333468, learning_rate=0.1413670964405497, max_depth=8, min_child_weight=35.77432109356246, n_estimators=16, reg_alpha=40.64238067539015, subsample=0.6009037853772013, total=   0.0s
[CV] colsample_bytree=0.9200931055241055, gamma=1.8046850899511557, learning_rate=0.3023836801358122, max_depth=30, min_child_weight=103.93311275507797, n_estimators=8, reg_alpha=87.23728073393639, subsample=0.9690801740743847 
[CV]  colsample_bytree=0.9200931055241055, gamma=1.8046850899511557, learning_rate=0.3023836801358122, max_depth=30, min_child_weight=103.93311275507797, n_estimators=8, reg_alpha=87.23728073393639, subsample=0.9690801740743847, total=   0.0s
[CV] colsample_bytree=0.9200931055241055, gamma=1.8046850899511557, learning_rate=0.3023836801358122, max_depth=30, min_child_weight=103.93311275507797, n_estimators=8, reg_alpha=87.23728073393639, subsample=0.9690801740743847 
[CV]  colsample_bytree=0.9200931055241055, gamma=1.8046850899511557, learning_rate=0.3023836801358122, max_depth=30, min_child_weight=103.93311275507797, n_estimators=8, reg_alpha=87.23728073393639, subsample=0.9690801740743847, total=   0.0s
[CV] colsample_bytree=0.9200931055241055, gamma=1.8046850899511557, learning_rate=0.3023836801358122, max_depth=30, min_child_weight=103.93311275507797, n_estimators=8, reg_alpha=87.23728073393639, subsample=0.9690801740743847 
[CV]  colsample_bytree=0.9200931055241055, gamma=1.8046850899511557, learning_rate=0.3023836801358122, max_depth=30, min_child_weight=103.93311275507797, n_estimators=8, reg_alpha=87.23728073393639, subsample=0.9690801740743847, total=   0.0s
[CV] colsample_bytree=0.8079425311998424, gamma=4.607424566663337, learning_rate=0.07851984345022736, max_depth=36, min_child_weight=47.08877215425798, n_estimators=37, reg_alpha=37.49844668093124, subsample=0.9672553496087561 
[CV]  colsample_bytree=0.8079425311998424, gamma=4.607424566663337, learning_rate=0.07851984345022736, max_depth=36, min_child_weight=47.08877215425798, n_estimators=37, reg_alpha=37.49844668093124, subsample=0.9672553496087561, total=   0.1s
[CV] colsample_bytree=0.8079425311998424, gamma=4.607424566663337, learning_rate=0.07851984345022736, max_depth=36, min_child_weight=47.08877215425798, n_estimators=37, reg_alpha=37.49844668093124, subsample=0.9672553496087561 
[CV]  colsample_bytree=0.8079425311998424, gamma=4.607424566663337, learning_rate=0.07851984345022736, max_depth=36, min_child_weight=47.08877215425798, n_estimators=37, reg_alpha=37.49844668093124, subsample=0.9672553496087561, total=   0.0s
[CV] colsample_bytree=0.8079425311998424, gamma=4.607424566663337, learning_rate=0.07851984345022736, max_depth=36, min_child_weight=47.08877215425798, n_estimators=37, reg_alpha=37.49844668093124, subsample=0.9672553496087561 
[CV]  colsample_bytree=0.8079425311998424, gamma=4.607424566663337, learning_rate=0.07851984345022736, max_depth=36, min_child_weight=47.08877215425798, n_estimators=37, reg_alpha=37.49844668093124, subsample=0.9672553496087561, total=   0.0s
[CV] colsample_bytree=0.9640945047746815, gamma=5.035095098483965, learning_rate=0.4363732260078889, max_depth=9, min_child_weight=12.74989540147968, n_estimators=31, reg_alpha=34.65059659524285, subsample=0.9799420324585474 
[CV]  colsample_bytree=0.9640945047746815, gamma=5.035095098483965, learning_rate=0.4363732260078889, max_depth=9, min_child_weight=12.74989540147968, n_estimators=31, reg_alpha=34.65059659524285, subsample=0.9799420324585474, total=   0.0s
[CV] colsample_bytree=0.9640945047746815, gamma=5.035095098483965, learning_rate=0.4363732260078889, max_depth=9, min_child_weight=12.74989540147968, n_estimators=31, reg_alpha=34.65059659524285, subsample=0.9799420324585474 
[CV]  colsample_bytree=0.9640945047746815, gamma=5.035095098483965, learning_rate=0.4363732260078889, max_depth=9, min_child_weight=12.74989540147968, n_estimators=31, reg_alpha=34.65059659524285, subsample=0.9799420324585474, total=   0.0s
[CV] colsample_bytree=0.9640945047746815, gamma=5.035095098483965, learning_rate=0.4363732260078889, max_depth=9, min_child_weight=12.74989540147968, n_estimators=31, reg_alpha=34.65059659524285, subsample=0.9799420324585474 
[CV]  colsample_bytree=0.9640945047746815, gamma=5.035095098483965, learning_rate=0.4363732260078889, max_depth=9, min_child_weight=12.74989540147968, n_estimators=31, reg_alpha=34.65059659524285, subsample=0.9799420324585474, total=   0.1s
[CV] colsample_bytree=0.9771631728950947, gamma=7.67777192147274, learning_rate=0.1661191988247473, max_depth=4, min_child_weight=58.526147523531904, n_estimators=26, reg_alpha=25.4296608655599, subsample=0.8742398615570541 
[CV]  colsample_bytree=0.9771631728950947, gamma=7.67777192147274, learning_rate=0.1661191988247473, max_depth=4, min_child_weight=58.526147523531904, n_estimators=26, reg_alpha=25.4296608655599, subsample=0.8742398615570541, total=   0.0s
[CV] colsample_bytree=0.9771631728950947, gamma=7.67777192147274, learning_rate=0.1661191988247473, max_depth=4, min_child_weight=58.526147523531904, n_estimators=26, reg_alpha=25.4296608655599, subsample=0.8742398615570541 
[CV]  colsample_bytree=0.9771631728950947, gamma=7.67777192147274, learning_rate=0.1661191988247473, max_depth=4, min_child_weight=58.526147523531904, n_estimators=26, reg_alpha=25.4296608655599, subsample=0.8742398615570541, total=   0.0s
[CV] colsample_bytree=0.9771631728950947, gamma=7.67777192147274, learning_rate=0.1661191988247473, max_depth=4, min_child_weight=58.526147523531904, n_estimators=26, reg_alpha=25.4296608655599, subsample=0.8742398615570541 
[CV]  colsample_bytree=0.9771631728950947, gamma=7.67777192147274, learning_rate=0.1661191988247473, max_depth=4, min_child_weight=58.526147523531904, n_estimators=26, reg_alpha=25.4296608655599, subsample=0.8742398615570541, total=   0.0s
[CV] colsample_bytree=0.8907849255573902, gamma=9.9346046309985, learning_rate=0.38379460560721185, max_depth=36, min_child_weight=125.20972702800735, n_estimators=8, reg_alpha=68.79385255352041, subsample=0.9781684065852686 
[CV]  colsample_bytree=0.8907849255573902, gamma=9.9346046309985, learning_rate=0.38379460560721185, max_depth=36, min_child_weight=125.20972702800735, n_estimators=8, reg_alpha=68.79385255352041, subsample=0.9781684065852686, total=   0.0s
[CV] colsample_bytree=0.8907849255573902, gamma=9.9346046309985, learning_rate=0.38379460560721185, max_depth=36, min_child_weight=125.20972702800735, n_estimators=8, reg_alpha=68.79385255352041, subsample=0.9781684065852686 
[CV]  colsample_bytree=0.8907849255573902, gamma=9.9346046309985, learning_rate=0.38379460560721185, max_depth=36, min_child_weight=125.20972702800735, n_estimators=8, reg_alpha=68.79385255352041, subsample=0.9781684065852686, total=   0.0s
[CV] colsample_bytree=0.8907849255573902, gamma=9.9346046309985, learning_rate=0.38379460560721185, max_depth=36, min_child_weight=125.20972702800735, n_estimators=8, reg_alpha=68.79385255352041, subsample=0.9781684065852686 
[CV]  colsample_bytree=0.8907849255573902, gamma=9.9346046309985, learning_rate=0.38379460560721185, max_depth=36, min_child_weight=125.20972702800735, n_estimators=8, reg_alpha=68.79385255352041, subsample=0.9781684065852686, total=   0.0s
[CV] colsample_bytree=0.929605962100854, gamma=2.5689351891454337, learning_rate=0.0882526543052029, max_depth=3, min_child_weight=15.253308260614421, n_estimators=29, reg_alpha=19.901743299224854, subsample=0.9808827113163352 
[CV]  colsample_bytree=0.929605962100854, gamma=2.5689351891454337, learning_rate=0.0882526543052029, max_depth=3, min_child_weight=15.253308260614421, n_estimators=29, reg_alpha=19.901743299224854, subsample=0.9808827113163352, total=   0.0s
[CV] colsample_bytree=0.929605962100854, gamma=2.5689351891454337, learning_rate=0.0882526543052029, max_depth=3, min_child_weight=15.253308260614421, n_estimators=29, reg_alpha=19.901743299224854, subsample=0.9808827113163352 
[CV]  colsample_bytree=0.929605962100854, gamma=2.5689351891454337, learning_rate=0.0882526543052029, max_depth=3, min_child_weight=15.253308260614421, n_estimators=29, reg_alpha=19.901743299224854, subsample=0.9808827113163352, total=   0.0s
[CV] colsample_bytree=0.929605962100854, gamma=2.5689351891454337, learning_rate=0.0882526543052029, max_depth=3, min_child_weight=15.253308260614421, n_estimators=29, reg_alpha=19.901743299224854, subsample=0.9808827113163352 
[CV]  colsample_bytree=0.929605962100854, gamma=2.5689351891454337, learning_rate=0.0882526543052029, max_depth=3, min_child_weight=15.253308260614421, n_estimators=29, reg_alpha=19.901743299224854, subsample=0.9808827113163352, total=   0.0s
[CV] colsample_bytree=0.7792425647722292, gamma=2.368687035569985, learning_rate=0.2044016044090382, max_depth=8, min_child_weight=37.306512981118345, n_estimators=12, reg_alpha=23.297831295710196, subsample=0.8954194786743525 
[CV]  colsample_bytree=0.7792425647722292, gamma=2.368687035569985, learning_rate=0.2044016044090382, max_depth=8, min_child_weight=37.306512981118345, n_estimators=12, reg_alpha=23.297831295710196, subsample=0.8954194786743525, total=   0.0s
[CV] colsample_bytree=0.7792425647722292, gamma=2.368687035569985, learning_rate=0.2044016044090382, max_depth=8, min_child_weight=37.306512981118345, n_estimators=12, reg_alpha=23.297831295710196, subsample=0.8954194786743525 
[CV]  colsample_bytree=0.7792425647722292, gamma=2.368687035569985, learning_rate=0.2044016044090382, max_depth=8, min_child_weight=37.306512981118345, n_estimators=12, reg_alpha=23.297831295710196, subsample=0.8954194786743525, total=   0.0s
[CV] colsample_bytree=0.7792425647722292, gamma=2.368687035569985, learning_rate=0.2044016044090382, max_depth=8, min_child_weight=37.306512981118345, n_estimators=12, reg_alpha=23.297831295710196, subsample=0.8954194786743525 
[CV]  colsample_bytree=0.7792425647722292, gamma=2.368687035569985, learning_rate=0.2044016044090382, max_depth=8, min_child_weight=37.306512981118345, n_estimators=12, reg_alpha=23.297831295710196, subsample=0.8954194786743525, total=   0.0s
[CV] colsample_bytree=0.9213491831089688, gamma=2.4979053418638895, learning_rate=0.35112873408705864, max_depth=13, min_child_weight=141.57781671037563, n_estimators=21, reg_alpha=74.46671546116669, subsample=0.6631160615502354 
[CV]  colsample_bytree=0.9213491831089688, gamma=2.4979053418638895, learning_rate=0.35112873408705864, max_depth=13, min_child_weight=141.57781671037563, n_estimators=21, reg_alpha=74.46671546116669, subsample=0.6631160615502354, total=   0.0s
[CV] colsample_bytree=0.9213491831089688, gamma=2.4979053418638895, learning_rate=0.35112873408705864, max_depth=13, min_child_weight=141.57781671037563, n_estimators=21, reg_alpha=74.46671546116669, subsample=0.6631160615502354 
[CV]  colsample_bytree=0.9213491831089688, gamma=2.4979053418638895, learning_rate=0.35112873408705864, max_depth=13, min_child_weight=141.57781671037563, n_estimators=21, reg_alpha=74.46671546116669, subsample=0.6631160615502354, total=   0.0s
[CV] colsample_bytree=0.9213491831089688, gamma=2.4979053418638895, learning_rate=0.35112873408705864, max_depth=13, min_child_weight=141.57781671037563, n_estimators=21, reg_alpha=74.46671546116669, subsample=0.6631160615502354 
[CV]  colsample_bytree=0.9213491831089688, gamma=2.4979053418638895, learning_rate=0.35112873408705864, max_depth=13, min_child_weight=141.57781671037563, n_estimators=21, reg_alpha=74.46671546116669, subsample=0.6631160615502354, total=   0.0s
[CV] colsample_bytree=0.9813081076844272, gamma=0.41475079576498874, learning_rate=0.17873151998809783, max_depth=3, min_child_weight=59.18488883891633, n_estimators=26, reg_alpha=14.464165778976199, subsample=0.9855798461133446 
[CV]  colsample_bytree=0.9813081076844272, gamma=0.41475079576498874, learning_rate=0.17873151998809783, max_depth=3, min_child_weight=59.18488883891633, n_estimators=26, reg_alpha=14.464165778976199, subsample=0.9855798461133446, total=   0.0s
[CV] colsample_bytree=0.9813081076844272, gamma=0.41475079576498874, learning_rate=0.17873151998809783, max_depth=3, min_child_weight=59.18488883891633, n_estimators=26, reg_alpha=14.464165778976199, subsample=0.9855798461133446 
[CV]  colsample_bytree=0.9813081076844272, gamma=0.41475079576498874, learning_rate=0.17873151998809783, max_depth=3, min_child_weight=59.18488883891633, n_estimators=26, reg_alpha=14.464165778976199, subsample=0.9855798461133446, total=   0.0s
[CV] colsample_bytree=0.9813081076844272, gamma=0.41475079576498874, learning_rate=0.17873151998809783, max_depth=3, min_child_weight=59.18488883891633, n_estimators=26, reg_alpha=14.464165778976199, subsample=0.9855798461133446 
[CV]  colsample_bytree=0.9813081076844272, gamma=0.41475079576498874, learning_rate=0.17873151998809783, max_depth=3, min_child_weight=59.18488883891633, n_estimators=26, reg_alpha=14.464165778976199, subsample=0.9855798461133446, total=   0.0s
[CV] colsample_bytree=0.9319809869450985, gamma=7.641630474650334, learning_rate=0.3763295340690517, max_depth=35, min_child_weight=48.046235077153966, n_estimators=27, reg_alpha=94.09530467636145, subsample=0.9980504267938731 
[CV]  colsample_bytree=0.9319809869450985, gamma=7.641630474650334, learning_rate=0.3763295340690517, max_depth=35, min_child_weight=48.046235077153966, n_estimators=27, reg_alpha=94.09530467636145, subsample=0.9980504267938731, total=   0.0s
[CV] colsample_bytree=0.9319809869450985, gamma=7.641630474650334, learning_rate=0.3763295340690517, max_depth=35, min_child_weight=48.046235077153966, n_estimators=27, reg_alpha=94.09530467636145, subsample=0.9980504267938731 
[CV]  colsample_bytree=0.9319809869450985, gamma=7.641630474650334, learning_rate=0.3763295340690517, max_depth=35, min_child_weight=48.046235077153966, n_estimators=27, reg_alpha=94.09530467636145, subsample=0.9980504267938731, total=   0.0s
[CV] colsample_bytree=0.9319809869450985, gamma=7.641630474650334, learning_rate=0.3763295340690517, max_depth=35, min_child_weight=48.046235077153966, n_estimators=27, reg_alpha=94.09530467636145, subsample=0.9980504267938731 
[CV]  colsample_bytree=0.9319809869450985, gamma=7.641630474650334, learning_rate=0.3763295340690517, max_depth=35, min_child_weight=48.046235077153966, n_estimators=27, reg_alpha=94.09530467636145, subsample=0.9980504267938731, total=   0.0s
[CV] colsample_bytree=0.7810472404244629, gamma=0.509123835168872, learning_rate=0.14402874563640583, max_depth=38, min_child_weight=3.3287041397476425, n_estimators=37, reg_alpha=17.081316004836246, subsample=0.9601700712015551 
[CV]  colsample_bytree=0.7810472404244629, gamma=0.509123835168872, learning_rate=0.14402874563640583, max_depth=38, min_child_weight=3.3287041397476425, n_estimators=37, reg_alpha=17.081316004836246, subsample=0.9601700712015551, total=   0.1s
[CV] colsample_bytree=0.7810472404244629, gamma=0.509123835168872, learning_rate=0.14402874563640583, max_depth=38, min_child_weight=3.3287041397476425, n_estimators=37, reg_alpha=17.081316004836246, subsample=0.9601700712015551 
[CV]  colsample_bytree=0.7810472404244629, gamma=0.509123835168872, learning_rate=0.14402874563640583, max_depth=38, min_child_weight=3.3287041397476425, n_estimators=37, reg_alpha=17.081316004836246, subsample=0.9601700712015551, total=   0.1s
[CV] colsample_bytree=0.7810472404244629, gamma=0.509123835168872, learning_rate=0.14402874563640583, max_depth=38, min_child_weight=3.3287041397476425, n_estimators=37, reg_alpha=17.081316004836246, subsample=0.9601700712015551 
[CV]  colsample_bytree=0.7810472404244629, gamma=0.509123835168872, learning_rate=0.14402874563640583, max_depth=38, min_child_weight=3.3287041397476425, n_estimators=37, reg_alpha=17.081316004836246, subsample=0.9601700712015551, total=   0.1s
[CV] colsample_bytree=0.9553548432244114, gamma=3.5496191113748434, learning_rate=0.21091093108795217, max_depth=38, min_child_weight=188.81084143186024, n_estimators=3, reg_alpha=61.14438395631305, subsample=0.9560652404397637 
[CV]  colsample_bytree=0.9553548432244114, gamma=3.5496191113748434, learning_rate=0.21091093108795217, max_depth=38, min_child_weight=188.81084143186024, n_estimators=3, reg_alpha=61.14438395631305, subsample=0.9560652404397637, total=   0.0s
[CV] colsample_bytree=0.9553548432244114, gamma=3.5496191113748434, learning_rate=0.21091093108795217, max_depth=38, min_child_weight=188.81084143186024, n_estimators=3, reg_alpha=61.14438395631305, subsample=0.9560652404397637 
[CV]  colsample_bytree=0.9553548432244114, gamma=3.5496191113748434, learning_rate=0.21091093108795217, max_depth=38, min_child_weight=188.81084143186024, n_estimators=3, reg_alpha=61.14438395631305, subsample=0.9560652404397637, total=   0.0s
[CV] colsample_bytree=0.9553548432244114, gamma=3.5496191113748434, learning_rate=0.21091093108795217, max_depth=38, min_child_weight=188.81084143186024, n_estimators=3, reg_alpha=61.14438395631305, subsample=0.9560652404397637 
[CV]  colsample_bytree=0.9553548432244114, gamma=3.5496191113748434, learning_rate=0.21091093108795217, max_depth=38, min_child_weight=188.81084143186024, n_estimators=3, reg_alpha=61.14438395631305, subsample=0.9560652404397637, total=   0.0s
[CV] colsample_bytree=0.8480014254137175, gamma=6.096583426365768, learning_rate=0.24870107988230855, max_depth=28, min_child_weight=14.613900380015398, n_estimators=39, reg_alpha=84.26715450016715, subsample=0.9888901564054718 
[CV]  colsample_bytree=0.8480014254137175, gamma=6.096583426365768, learning_rate=0.24870107988230855, max_depth=28, min_child_weight=14.613900380015398, n_estimators=39, reg_alpha=84.26715450016715, subsample=0.9888901564054718, total=   0.0s
[CV] colsample_bytree=0.8480014254137175, gamma=6.096583426365768, learning_rate=0.24870107988230855, max_depth=28, min_child_weight=14.613900380015398, n_estimators=39, reg_alpha=84.26715450016715, subsample=0.9888901564054718 
[CV]  colsample_bytree=0.8480014254137175, gamma=6.096583426365768, learning_rate=0.24870107988230855, max_depth=28, min_child_weight=14.613900380015398, n_estimators=39, reg_alpha=84.26715450016715, subsample=0.9888901564054718, total=   0.0s
[CV] colsample_bytree=0.8480014254137175, gamma=6.096583426365768, learning_rate=0.24870107988230855, max_depth=28, min_child_weight=14.613900380015398, n_estimators=39, reg_alpha=84.26715450016715, subsample=0.9888901564054718 
[CV]  colsample_bytree=0.8480014254137175, gamma=6.096583426365768, learning_rate=0.24870107988230855, max_depth=28, min_child_weight=14.613900380015398, n_estimators=39, reg_alpha=84.26715450016715, subsample=0.9888901564054718, total=   0.0s
[CV] colsample_bytree=0.9744058218541515, gamma=2.9507857524949586, learning_rate=0.3990744659677848, max_depth=23, min_child_weight=3.329979147595724, n_estimators=19, reg_alpha=4.127239852707413, subsample=0.9318278660909255 
[CV]  colsample_bytree=0.9744058218541515, gamma=2.9507857524949586, learning_rate=0.3990744659677848, max_depth=23, min_child_weight=3.329979147595724, n_estimators=19, reg_alpha=4.127239852707413, subsample=0.9318278660909255, total=   0.0s
[CV] colsample_bytree=0.9744058218541515, gamma=2.9507857524949586, learning_rate=0.3990744659677848, max_depth=23, min_child_weight=3.329979147595724, n_estimators=19, reg_alpha=4.127239852707413, subsample=0.9318278660909255 
[CV]  colsample_bytree=0.9744058218541515, gamma=2.9507857524949586, learning_rate=0.3990744659677848, max_depth=23, min_child_weight=3.329979147595724, n_estimators=19, reg_alpha=4.127239852707413, subsample=0.9318278660909255, total=   0.0s
[CV] colsample_bytree=0.9744058218541515, gamma=2.9507857524949586, learning_rate=0.3990744659677848, max_depth=23, min_child_weight=3.329979147595724, n_estimators=19, reg_alpha=4.127239852707413, subsample=0.9318278660909255 
[CV]  colsample_bytree=0.9744058218541515, gamma=2.9507857524949586, learning_rate=0.3990744659677848, max_depth=23, min_child_weight=3.329979147595724, n_estimators=19, reg_alpha=4.127239852707413, subsample=0.9318278660909255, total=   0.0s
[CV] colsample_bytree=0.9925474376498676, gamma=5.115972970056662, learning_rate=0.12378024542969297, max_depth=32, min_child_weight=107.60298089179803, n_estimators=13, reg_alpha=19.001725628050046, subsample=0.9079973453605156 
[CV]  colsample_bytree=0.9925474376498676, gamma=5.115972970056662, learning_rate=0.12378024542969297, max_depth=32, min_child_weight=107.60298089179803, n_estimators=13, reg_alpha=19.001725628050046, subsample=0.9079973453605156, total=   0.0s
[CV] colsample_bytree=0.9925474376498676, gamma=5.115972970056662, learning_rate=0.12378024542969297, max_depth=32, min_child_weight=107.60298089179803, n_estimators=13, reg_alpha=19.001725628050046, subsample=0.9079973453605156 
[CV]  colsample_bytree=0.9925474376498676, gamma=5.115972970056662, learning_rate=0.12378024542969297, max_depth=32, min_child_weight=107.60298089179803, n_estimators=13, reg_alpha=19.001725628050046, subsample=0.9079973453605156, total=   0.0s
[CV] colsample_bytree=0.9925474376498676, gamma=5.115972970056662, learning_rate=0.12378024542969297, max_depth=32, min_child_weight=107.60298089179803, n_estimators=13, reg_alpha=19.001725628050046, subsample=0.9079973453605156 
[CV]  colsample_bytree=0.9925474376498676, gamma=5.115972970056662, learning_rate=0.12378024542969297, max_depth=32, min_child_weight=107.60298089179803, n_estimators=13, reg_alpha=19.001725628050046, subsample=0.9079973453605156, total=   0.0s
[CV] colsample_bytree=0.9470955032103993, gamma=8.472394764586063, learning_rate=0.3956872015105697, max_depth=12, min_child_weight=49.097485232345235, n_estimators=19, reg_alpha=45.68785832483281, subsample=0.9943005201213969 
[CV]  colsample_bytree=0.9470955032103993, gamma=8.472394764586063, learning_rate=0.3956872015105697, max_depth=12, min_child_weight=49.097485232345235, n_estimators=19, reg_alpha=45.68785832483281, subsample=0.9943005201213969, total=   0.0s
[CV] colsample_bytree=0.9470955032103993, gamma=8.472394764586063, learning_rate=0.3956872015105697, max_depth=12, min_child_weight=49.097485232345235, n_estimators=19, reg_alpha=45.68785832483281, subsample=0.9943005201213969 
[CV]  colsample_bytree=0.9470955032103993, gamma=8.472394764586063, learning_rate=0.3956872015105697, max_depth=12, min_child_weight=49.097485232345235, n_estimators=19, reg_alpha=45.68785832483281, subsample=0.9943005201213969, total=   0.0s
[CV] colsample_bytree=0.9470955032103993, gamma=8.472394764586063, learning_rate=0.3956872015105697, max_depth=12, min_child_weight=49.097485232345235, n_estimators=19, reg_alpha=45.68785832483281, subsample=0.9943005201213969 
[CV]  colsample_bytree=0.9470955032103993, gamma=8.472394764586063, learning_rate=0.3956872015105697, max_depth=12, min_child_weight=49.097485232345235, n_estimators=19, reg_alpha=45.68785832483281, subsample=0.9943005201213969, total=   0.0s
[CV] colsample_bytree=0.9987744424423294, gamma=3.2825610365078086, learning_rate=0.4170425390577573, max_depth=12, min_child_weight=52.553403831939406, n_estimators=15, reg_alpha=46.40198056641713, subsample=0.8276000392473873 
[CV]  colsample_bytree=0.9987744424423294, gamma=3.2825610365078086, learning_rate=0.4170425390577573, max_depth=12, min_child_weight=52.553403831939406, n_estimators=15, reg_alpha=46.40198056641713, subsample=0.8276000392473873, total=   0.0s
[CV] colsample_bytree=0.9987744424423294, gamma=3.2825610365078086, learning_rate=0.4170425390577573, max_depth=12, min_child_weight=52.553403831939406, n_estimators=15, reg_alpha=46.40198056641713, subsample=0.8276000392473873 
[CV]  colsample_bytree=0.9987744424423294, gamma=3.2825610365078086, learning_rate=0.4170425390577573, max_depth=12, min_child_weight=52.553403831939406, n_estimators=15, reg_alpha=46.40198056641713, subsample=0.8276000392473873, total=   0.0s
[CV] colsample_bytree=0.9987744424423294, gamma=3.2825610365078086, learning_rate=0.4170425390577573, max_depth=12, min_child_weight=52.553403831939406, n_estimators=15, reg_alpha=46.40198056641713, subsample=0.8276000392473873 
[CV]  colsample_bytree=0.9987744424423294, gamma=3.2825610365078086, learning_rate=0.4170425390577573, max_depth=12, min_child_weight=52.553403831939406, n_estimators=15, reg_alpha=46.40198056641713, subsample=0.8276000392473873, total=   0.0s
[CV] colsample_bytree=0.9279049918335398, gamma=1.267357091420086, learning_rate=0.2859294592439557, max_depth=36, min_child_weight=11.168561137155821, n_estimators=18, reg_alpha=53.49130997158936, subsample=0.8160414295335819 
[CV]  colsample_bytree=0.9279049918335398, gamma=1.267357091420086, learning_rate=0.2859294592439557, max_depth=36, min_child_weight=11.168561137155821, n_estimators=18, reg_alpha=53.49130997158936, subsample=0.8160414295335819, total=   0.0s
[CV] colsample_bytree=0.9279049918335398, gamma=1.267357091420086, learning_rate=0.2859294592439557, max_depth=36, min_child_weight=11.168561137155821, n_estimators=18, reg_alpha=53.49130997158936, subsample=0.8160414295335819 
[CV]  colsample_bytree=0.9279049918335398, gamma=1.267357091420086, learning_rate=0.2859294592439557, max_depth=36, min_child_weight=11.168561137155821, n_estimators=18, reg_alpha=53.49130997158936, subsample=0.8160414295335819, total=   0.0s
[CV] colsample_bytree=0.9279049918335398, gamma=1.267357091420086, learning_rate=0.2859294592439557, max_depth=36, min_child_weight=11.168561137155821, n_estimators=18, reg_alpha=53.49130997158936, subsample=0.8160414295335819 
[CV]  colsample_bytree=0.9279049918335398, gamma=1.267357091420086, learning_rate=0.2859294592439557, max_depth=36, min_child_weight=11.168561137155821, n_estimators=18, reg_alpha=53.49130997158936, subsample=0.8160414295335819, total=   0.0s
[CV] colsample_bytree=0.9217589706442475, gamma=2.0211656769454045, learning_rate=0.39132388667670553, max_depth=23, min_child_weight=44.321005016727135, n_estimators=32, reg_alpha=33.20029808377724, subsample=0.8468330105596066 
[CV]  colsample_bytree=0.9217589706442475, gamma=2.0211656769454045, learning_rate=0.39132388667670553, max_depth=23, min_child_weight=44.321005016727135, n_estimators=32, reg_alpha=33.20029808377724, subsample=0.8468330105596066, total=   0.0s
[CV] colsample_bytree=0.9217589706442475, gamma=2.0211656769454045, learning_rate=0.39132388667670553, max_depth=23, min_child_weight=44.321005016727135, n_estimators=32, reg_alpha=33.20029808377724, subsample=0.8468330105596066 
[CV]  colsample_bytree=0.9217589706442475, gamma=2.0211656769454045, learning_rate=0.39132388667670553, max_depth=23, min_child_weight=44.321005016727135, n_estimators=32, reg_alpha=33.20029808377724, subsample=0.8468330105596066, total=   0.0s
[CV] colsample_bytree=0.9217589706442475, gamma=2.0211656769454045, learning_rate=0.39132388667670553, max_depth=23, min_child_weight=44.321005016727135, n_estimators=32, reg_alpha=33.20029808377724, subsample=0.8468330105596066 
[CV]  colsample_bytree=0.9217589706442475, gamma=2.0211656769454045, learning_rate=0.39132388667670553, max_depth=23, min_child_weight=44.321005016727135, n_estimators=32, reg_alpha=33.20029808377724, subsample=0.8468330105596066, total=   0.0s
[CV] colsample_bytree=0.9370726723194647, gamma=1.2890024298161562, learning_rate=0.17351709628123335, max_depth=6, min_child_weight=32.85969411653827, n_estimators=16, reg_alpha=55.80918587730207, subsample=0.8467496210296448 
[CV]  colsample_bytree=0.9370726723194647, gamma=1.2890024298161562, learning_rate=0.17351709628123335, max_depth=6, min_child_weight=32.85969411653827, n_estimators=16, reg_alpha=55.80918587730207, subsample=0.8467496210296448, total=   0.0s
[CV] colsample_bytree=0.9370726723194647, gamma=1.2890024298161562, learning_rate=0.17351709628123335, max_depth=6, min_child_weight=32.85969411653827, n_estimators=16, reg_alpha=55.80918587730207, subsample=0.8467496210296448 
[CV]  colsample_bytree=0.9370726723194647, gamma=1.2890024298161562, learning_rate=0.17351709628123335, max_depth=6, min_child_weight=32.85969411653827, n_estimators=16, reg_alpha=55.80918587730207, subsample=0.8467496210296448, total=   0.0s
[CV] colsample_bytree=0.9370726723194647, gamma=1.2890024298161562, learning_rate=0.17351709628123335, max_depth=6, min_child_weight=32.85969411653827, n_estimators=16, reg_alpha=55.80918587730207, subsample=0.8467496210296448 
[CV]  colsample_bytree=0.9370726723194647, gamma=1.2890024298161562, learning_rate=0.17351709628123335, max_depth=6, min_child_weight=32.85969411653827, n_estimators=16, reg_alpha=55.80918587730207, subsample=0.8467496210296448, total=   0.0s
[CV] colsample_bytree=0.9882326458866737, gamma=7.60802456305756, learning_rate=0.3624183288775828, max_depth=28, min_child_weight=52.11635668562104, n_estimators=38, reg_alpha=47.56023651231594, subsample=0.9457133991353094 
[CV]  colsample_bytree=0.9882326458866737, gamma=7.60802456305756, learning_rate=0.3624183288775828, max_depth=28, min_child_weight=52.11635668562104, n_estimators=38, reg_alpha=47.56023651231594, subsample=0.9457133991353094, total=   0.0s
[CV] colsample_bytree=0.9882326458866737, gamma=7.60802456305756, learning_rate=0.3624183288775828, max_depth=28, min_child_weight=52.11635668562104, n_estimators=38, reg_alpha=47.56023651231594, subsample=0.9457133991353094 
[CV]  colsample_bytree=0.9882326458866737, gamma=7.60802456305756, learning_rate=0.3624183288775828, max_depth=28, min_child_weight=52.11635668562104, n_estimators=38, reg_alpha=47.56023651231594, subsample=0.9457133991353094, total=   0.1s
[CV] colsample_bytree=0.9882326458866737, gamma=7.60802456305756, learning_rate=0.3624183288775828, max_depth=28, min_child_weight=52.11635668562104, n_estimators=38, reg_alpha=47.56023651231594, subsample=0.9457133991353094 
[CV]  colsample_bytree=0.9882326458866737, gamma=7.60802456305756, learning_rate=0.3624183288775828, max_depth=28, min_child_weight=52.11635668562104, n_estimators=38, reg_alpha=47.56023651231594, subsample=0.9457133991353094, total=   0.1s
[CV] colsample_bytree=0.9956221844796125, gamma=4.255714499580487, learning_rate=0.4331346701783154, max_depth=22, min_child_weight=135.15184636283593, n_estimators=3, reg_alpha=45.769058335664084, subsample=0.9526902894798567 
[CV]  colsample_bytree=0.9956221844796125, gamma=4.255714499580487, learning_rate=0.4331346701783154, max_depth=22, min_child_weight=135.15184636283593, n_estimators=3, reg_alpha=45.769058335664084, subsample=0.9526902894798567, total=   0.0s
[CV] colsample_bytree=0.9956221844796125, gamma=4.255714499580487, learning_rate=0.4331346701783154, max_depth=22, min_child_weight=135.15184636283593, n_estimators=3, reg_alpha=45.769058335664084, subsample=0.9526902894798567 
[CV]  colsample_bytree=0.9956221844796125, gamma=4.255714499580487, learning_rate=0.4331346701783154, max_depth=22, min_child_weight=135.15184636283593, n_estimators=3, reg_alpha=45.769058335664084, subsample=0.9526902894798567, total=   0.0s
[CV] colsample_bytree=0.9956221844796125, gamma=4.255714499580487, learning_rate=0.4331346701783154, max_depth=22, min_child_weight=135.15184636283593, n_estimators=3, reg_alpha=45.769058335664084, subsample=0.9526902894798567 
[CV]  colsample_bytree=0.9956221844796125, gamma=4.255714499580487, learning_rate=0.4331346701783154, max_depth=22, min_child_weight=135.15184636283593, n_estimators=3, reg_alpha=45.769058335664084, subsample=0.9526902894798567, total=   0.0s
[CV] colsample_bytree=0.9764741539362997, gamma=0.254762357788203, learning_rate=0.3417590793024611, max_depth=32, min_child_weight=80.69752871068955, n_estimators=26, reg_alpha=27.96950731543939, subsample=0.892989943330885 
[CV]  colsample_bytree=0.9764741539362997, gamma=0.254762357788203, learning_rate=0.3417590793024611, max_depth=32, min_child_weight=80.69752871068955, n_estimators=26, reg_alpha=27.96950731543939, subsample=0.892989943330885, total=   0.0s
[CV] colsample_bytree=0.9764741539362997, gamma=0.254762357788203, learning_rate=0.3417590793024611, max_depth=32, min_child_weight=80.69752871068955, n_estimators=26, reg_alpha=27.96950731543939, subsample=0.892989943330885 
[CV]  colsample_bytree=0.9764741539362997, gamma=0.254762357788203, learning_rate=0.3417590793024611, max_depth=32, min_child_weight=80.69752871068955, n_estimators=26, reg_alpha=27.96950731543939, subsample=0.892989943330885, total=   0.0s
[CV] colsample_bytree=0.9764741539362997, gamma=0.254762357788203, learning_rate=0.3417590793024611, max_depth=32, min_child_weight=80.69752871068955, n_estimators=26, reg_alpha=27.96950731543939, subsample=0.892989943330885 
[CV]  colsample_bytree=0.9764741539362997, gamma=0.254762357788203, learning_rate=0.3417590793024611, max_depth=32, min_child_weight=80.69752871068955, n_estimators=26, reg_alpha=27.96950731543939, subsample=0.892989943330885, total=   0.0s
[CV] colsample_bytree=0.7301234216930994, gamma=3.0499397583521604, learning_rate=0.39837824366442137, max_depth=21, min_child_weight=20.012279875342234, n_estimators=16, reg_alpha=114.74123889704305, subsample=0.8370860677564377 
[CV]  colsample_bytree=0.7301234216930994, gamma=3.0499397583521604, learning_rate=0.39837824366442137, max_depth=21, min_child_weight=20.012279875342234, n_estimators=16, reg_alpha=114.74123889704305, subsample=0.8370860677564377, total=   0.0s
[CV] colsample_bytree=0.7301234216930994, gamma=3.0499397583521604, learning_rate=0.39837824366442137, max_depth=21, min_child_weight=20.012279875342234, n_estimators=16, reg_alpha=114.74123889704305, subsample=0.8370860677564377 
[CV]  colsample_bytree=0.7301234216930994, gamma=3.0499397583521604, learning_rate=0.39837824366442137, max_depth=21, min_child_weight=20.012279875342234, n_estimators=16, reg_alpha=114.74123889704305, subsample=0.8370860677564377, total=   0.0s
[CV] colsample_bytree=0.7301234216930994, gamma=3.0499397583521604, learning_rate=0.39837824366442137, max_depth=21, min_child_weight=20.012279875342234, n_estimators=16, reg_alpha=114.74123889704305, subsample=0.8370860677564377 
[CV]  colsample_bytree=0.7301234216930994, gamma=3.0499397583521604, learning_rate=0.39837824366442137, max_depth=21, min_child_weight=20.012279875342234, n_estimators=16, reg_alpha=114.74123889704305, subsample=0.8370860677564377, total=   0.0s
[CV] colsample_bytree=0.9628482146099234, gamma=1.51375569992799, learning_rate=0.30414838459538135, max_depth=38, min_child_weight=86.25629386661218, n_estimators=16, reg_alpha=156.5220862826021, subsample=0.9750849728808573 
[CV]  colsample_bytree=0.9628482146099234, gamma=1.51375569992799, learning_rate=0.30414838459538135, max_depth=38, min_child_weight=86.25629386661218, n_estimators=16, reg_alpha=156.5220862826021, subsample=0.9750849728808573, total=   0.0s
[CV] colsample_bytree=0.9628482146099234, gamma=1.51375569992799, learning_rate=0.30414838459538135, max_depth=38, min_child_weight=86.25629386661218, n_estimators=16, reg_alpha=156.5220862826021, subsample=0.9750849728808573 
[CV]  colsample_bytree=0.9628482146099234, gamma=1.51375569992799, learning_rate=0.30414838459538135, max_depth=38, min_child_weight=86.25629386661218, n_estimators=16, reg_alpha=156.5220862826021, subsample=0.9750849728808573, total=   0.0s
[CV] colsample_bytree=0.9628482146099234, gamma=1.51375569992799, learning_rate=0.30414838459538135, max_depth=38, min_child_weight=86.25629386661218, n_estimators=16, reg_alpha=156.5220862826021, subsample=0.9750849728808573 
[CV]  colsample_bytree=0.9628482146099234, gamma=1.51375569992799, learning_rate=0.30414838459538135, max_depth=38, min_child_weight=86.25629386661218, n_estimators=16, reg_alpha=156.5220862826021, subsample=0.9750849728808573, total=   0.0s
[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    9.5s finished
Out[88]:
RandomizedSearchCV(cv=3,
                   estimator=XGBRegressor(base_score=None, booster=None,
                                          colsample_bylevel=None,
                                          colsample_bynode=None,
                                          colsample_bytree=None, gamma=None,
                                          gpu_id=None, importance_type='gain',
                                          interaction_constraints=None,
                                          learning_rate=None,
                                          max_delta_step=None, max_depth=None,
                                          min_child_weight=None, missing=nan,
                                          monotone_constraints=None,
                                          n_estimators=100, n...
                                        'min_child_weight': <scipy.stats._distn_infrastructure.rv_frozen object at 0x0000019F745E18B0>,
                                        'n_estimators': <scipy.stats._distn_infrastructure.rv_frozen object at 0x0000019F745E1880>,
                                        'reg_alpha': <scipy.stats._distn_infrastructure.rv_frozen object at 0x0000019F745E18B0>,
                                        'subsample': <scipy.stats._distn_infrastructure.rv_frozen object at 0x0000019F742DFEE0>},
                   random_state=1, verbose=2)
In [89]:
xgbrs.best_params_
Out[89]:
{'colsample_bytree': 0.8095793735045576,
 'gamma': 4.784503456129699,
 'learning_rate': 0.20966135876466158,
 'max_depth': 25,
 'min_child_weight': 12.991263135531565,
 'n_estimators': 34,
 'reg_alpha': 7.466469039520882,
 'subsample': 0.9699481172402621}
In [90]:
xgb_best_random = xgbrs.best_estimator_

xgb_rscv_train = xgb_best_random.score(x_train, y_train.values.ravel())
xgb_rscv_test = xgb_best_random.score(x_test, y_test.values.ravel())

xgb_rscv = cross_validate(xgb_best_random, x, y.values.ravel(), cv=10)
In [91]:
model_comparison['XGBoost Randomized Search CV'] = pd.Series(data=[rf_rscv_train, rf_rscv_test, xgb_rscv['test_score'].mean(), xgb_rscv['test_score'].std()],index=labels)

model_comparison.T
Out[91]:
Training Score Testing Score 10-Fold CV Mean 10-Fold CV Std. Dev.
Simple Linear Regression 0.645278 0.648877 0.622869 0.076005
Decision Tree 0.994855 0.846844 0.861023 0.048994
Random Forest 0.982552 0.909249 0.919497 0.024503
Random Forest with MinMaxScaler 0.982306 0.906661 0.920405 0.023778
Gradient Boosting Regressor 0.952045 0.899542 0.904101 0.022228
Randomized Search CV 0.983311 0.905970 0.917427 0.024922
Grid Search CV 0.983307 0.905936 0.917525 0.024961
XGBoost 0.994335 0.916113 0.933395 0.027554
XGBoost Randomized Search CV 0.983311 0.905970 0.929174 0.019948
In [92]:
xgb_grid_base = XGBRegressor(random_state=1)
In [93]:
grid_params = {
    'colsample_bytree': [.86,.87,.88],
    'gamma': [7.5,7.6,7.7],
    'learning_rate': [.1,.15,.2,.25,.3],
    'max_depth': [13],
    'min_child_weight': [5.6,5.7,5.8],
    'n_estimators': [37],
    'reg_alpha': [23,23.5,24,24.5],
    'subsample': [.8,.9,1]
}

xgb_gscv = GridSearchCV(estimator=xgb_grid_base, param_grid=grid_params, cv=3, n_jobs=-1, verbose=2)
In [94]:
xgb_gscv.fit(x_train, y_train.values.ravel())
Fitting 3 folds for each of 1620 candidates, totalling 4860 fits
[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.
[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:    2.7s
[Parallel(n_jobs=-1)]: Done 468 tasks      | elapsed:   10.8s
[Parallel(n_jobs=-1)]: Done 1280 tasks      | elapsed:   24.4s
[Parallel(n_jobs=-1)]: Done 2412 tasks      | elapsed:   42.7s
[Parallel(n_jobs=-1)]: Done 3872 tasks      | elapsed:  1.1min
[Parallel(n_jobs=-1)]: Done 4860 out of 4860 | elapsed:  1.4min finished
Out[94]:
GridSearchCV(cv=3,
             estimator=XGBRegressor(base_score=None, booster=None,
                                    colsample_bylevel=None,
                                    colsample_bynode=None,
                                    colsample_bytree=None, gamma=None,
                                    gpu_id=None, importance_type='gain',
                                    interaction_constraints=None,
                                    learning_rate=None, max_delta_step=None,
                                    max_depth=None, min_child_weight=None,
                                    missing=nan, monotone_constraints=None,
                                    n_estimators=100, n_jobs=...
                                    scale_pos_weight=None, subsample=None,
                                    tree_method=None, validate_parameters=None,
                                    verbosity=None),
             n_jobs=-1,
             param_grid={'colsample_bytree': [0.86, 0.87, 0.88],
                         'gamma': [7.5, 7.6, 7.7],
                         'learning_rate': [0.1, 0.15, 0.2, 0.25, 0.3],
                         'max_depth': [13], 'min_child_weight': [5.6, 5.7, 5.8],
                         'n_estimators': [37],
                         'reg_alpha': [23, 23.5, 24, 24.5],
                         'subsample': [0.8, 0.9, 1]},
             verbose=2)
In [95]:
xgbgs_best = xgb_gscv.best_estimator_

xgb_gscv_train = xgbgs_best.score(x_train, y_train.values.ravel())
xgb_gscv_test = xgbgs_best.score(x_test, y_test.values.ravel())

xgbgs_cv = cross_validate(xgbgs_best, x, y.values.ravel(), cv=10)
In [96]:
model_comparison['XGBoost Grid Search CV'] = pd.Series(data=[rf_gscv_train, rf_gscv_test, xgbgs_cv['test_score'].mean(), xgbgs_cv['test_score'].std()],
                                                       index=labels)

model_comparison.T
Out[96]:
Training Score Testing Score 10-Fold CV Mean 10-Fold CV Std. Dev.
Simple Linear Regression 0.645278 0.648877 0.622869 0.076005
Decision Tree 0.994855 0.846844 0.861023 0.048994
Random Forest 0.982552 0.909249 0.919497 0.024503
Random Forest with MinMaxScaler 0.982306 0.906661 0.920405 0.023778
Gradient Boosting Regressor 0.952045 0.899542 0.904101 0.022228
Randomized Search CV 0.983311 0.905970 0.917427 0.024922
Grid Search CV 0.983307 0.905936 0.917525 0.024961
XGBoost 0.994335 0.916113 0.933395 0.027554
XGBoost Randomized Search CV 0.983311 0.905970 0.929174 0.019948
XGBoost Grid Search CV 0.983307 0.905936 0.922162 0.023979